Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -34,6 +34,13 @@ function resolveEnvFile(mode: NodeEnvMode): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function loadDotenvFile(filePath: string, label: string): void {
|
||||
const result = dotenv.config({ path: filePath, override: false });
|
||||
if (result.error) {
|
||||
logger.warn(`[Env] Failed to load ${label} (${filePath}): ${result.error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function loadEnvironmentFromNodeEnv(): void {
|
||||
if (loaded) return;
|
||||
loaded = true;
|
||||
@@ -47,9 +54,5 @@ export function loadEnvironmentFromNodeEnv(): void {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = dotenv.config({ path: envFile, override: false });
|
||||
if (result.error) {
|
||||
logger.warn(`[Env] Failed to load ${envFile}: ${result.error.message}`);
|
||||
return;
|
||||
}
|
||||
loadDotenvFile(envFile, `env-vars/${mode}.env`);
|
||||
}
|
||||
@@ -188,6 +188,14 @@ export class GarnetClient {
|
||||
await this.getClient().hSet(key, fields);
|
||||
}
|
||||
|
||||
async expireKey(key: string, ttlSeconds: number): Promise<boolean> {
|
||||
if (ttlSeconds <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return Boolean(await this.getClient().expire(key, ttlSeconds));
|
||||
}
|
||||
|
||||
async appendStream(
|
||||
key: string,
|
||||
fields: Record<string, string>,
|
||||
@@ -207,6 +215,31 @@ export class GarnetClient {
|
||||
return this.getClient().xAdd(key, "*", fields, trimOptions);
|
||||
}
|
||||
|
||||
async readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
const rangeOptions =
|
||||
options?.count && options.count > 0
|
||||
? {
|
||||
COUNT: options.count,
|
||||
}
|
||||
: undefined;
|
||||
const messages = await this.getClient().xRange(
|
||||
key,
|
||||
start,
|
||||
end,
|
||||
rangeOptions,
|
||||
);
|
||||
|
||||
return messages.map((message) => ({
|
||||
id: String(message.id),
|
||||
message: normalizeStreamFields(message.message),
|
||||
}));
|
||||
}
|
||||
|
||||
async createConsumerGroup(
|
||||
key: string,
|
||||
group: string,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
export const KNOWN_SECRET_KEYS = [
|
||||
"INFISICAL_CLIENT_ID",
|
||||
"INFISICAL_CLIENT_SECRET",
|
||||
"NATS_ENDPOINT",
|
||||
"NATS_PASS",
|
||||
"NATS_USER",
|
||||
|
||||
@@ -7,7 +7,13 @@ export const DISCORD_MEMBER_LOOKUP_SUBJECT =
|
||||
"lolympus.discord.lookup-members";
|
||||
|
||||
export interface AlertPayload {
|
||||
kind: "latency" | "eval-score-drop" | "llm-error-rate" | "endpoint-down";
|
||||
kind:
|
||||
| "archive-stalled"
|
||||
| "endpoint-down"
|
||||
| "eval-drift"
|
||||
| "eval-score-drop"
|
||||
| "latency"
|
||||
| "llm-error-rate";
|
||||
metric: string;
|
||||
severity: "info" | "warn" | "critical";
|
||||
suggestedAction?: string | undefined;
|
||||
|
||||
+186
-11
@@ -1,10 +1,19 @@
|
||||
import { InfisicalSDK } from "@infisical/sdk";
|
||||
import { KNOWN_SECRET_KEYS, type KnownSecretKey } from "./known-secret-keys.js";
|
||||
|
||||
export type SecretMap = Partial<Record<KnownSecretKey, string>>;
|
||||
export type RawSecretMap = Record<string, string>;
|
||||
|
||||
const INFISICAL_SITE_URL = "https://secret-manager.dahyeon.us/";
|
||||
const KNOWN_SECRET_KEY_SET = new Set<string>(KNOWN_SECRET_KEYS);
|
||||
const IMDS_TOKEN_TTL_SECONDS = 60;
|
||||
const DEFAULT_IMDS_TIMEOUT_MS = 1_500;
|
||||
const DEFAULT_LOCAL_IMDS_BASE_URL = "http://127.0.0.1:8675";
|
||||
const DEFAULT_LINK_LOCAL_IMDS_BASE_URL = "http://169.254.169.254";
|
||||
const IMDS_TOKEN_TTL_HEADER = "X-aws-ec2-metadata-token-ttl-seconds";
|
||||
const IMDS_TOKEN_HEADER = "X-aws-ec2-metadata-token";
|
||||
const INFISICAL_CLIENT_ID_KEY = "INFISICAL_CLIENT_ID";
|
||||
const INFISICAL_CLIENT_SECRET_KEY = "INFISICAL_CLIENT_SECRET";
|
||||
|
||||
const infisicalClient = new InfisicalSDK({ siteUrl: INFISICAL_SITE_URL });
|
||||
|
||||
@@ -15,18 +24,10 @@ function isKnownSecretKey(value: string): value is KnownSecretKey {
|
||||
return KNOWN_SECRET_KEY_SET.has(value);
|
||||
}
|
||||
|
||||
function getInfisicalCredentials(): {
|
||||
interface InfisicalCredentials {
|
||||
clientSecret: string;
|
||||
clientId: string;
|
||||
projectId: string;
|
||||
} | null {
|
||||
const clientSecret = process.env["INFISICAL_CLIENT_SECRET"] ?? "";
|
||||
const clientId = process.env["INFISICAL_CLIENT_ID"] ?? "";
|
||||
const projectId = process.env["INFISICAL_PROJECT_ID"] ?? "";
|
||||
if (!clientSecret || !clientId || !projectId) {
|
||||
return null;
|
||||
}
|
||||
return { clientSecret, clientId, projectId };
|
||||
}
|
||||
|
||||
function getInfisicalEnvironment(): "dev" | "staging" | "prod" {
|
||||
@@ -35,6 +36,173 @@ function getInfisicalEnvironment(): "dev" | "staging" | "prod" {
|
||||
return "dev";
|
||||
}
|
||||
|
||||
function isDisabled(value: string | undefined): boolean {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
return normalized === "1" || normalized === "true" || normalized === "yes";
|
||||
}
|
||||
|
||||
function getImdsBaseUrls(): string[] {
|
||||
if (
|
||||
isDisabled(process.env["IMDS_DISABLED"]) ||
|
||||
isDisabled(process.env["METADATA_DISABLED"])
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const configured =
|
||||
process.env["METADATA_BASE_URL"]?.trim() ||
|
||||
process.env["IMDS_BASE_URL"]?.trim() ||
|
||||
process.env["MACHINE_ENV_BASE_URL"]?.trim();
|
||||
if (configured) {
|
||||
return [configured.replace(/\/+$/u, "")];
|
||||
}
|
||||
|
||||
const candidates =
|
||||
process.env["NODE_ENV"] === "prod"
|
||||
? [DEFAULT_LINK_LOCAL_IMDS_BASE_URL, DEFAULT_LOCAL_IMDS_BASE_URL]
|
||||
: [DEFAULT_LOCAL_IMDS_BASE_URL, DEFAULT_LINK_LOCAL_IMDS_BASE_URL];
|
||||
return [...new Set(candidates)];
|
||||
}
|
||||
|
||||
function getImdsTimeoutMs(): number {
|
||||
const raw =
|
||||
process.env["METADATA_TIMEOUT_MS"]?.trim() ||
|
||||
process.env["IMDS_TIMEOUT_MS"]?.trim();
|
||||
if (!raw) {
|
||||
return DEFAULT_IMDS_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0
|
||||
? parsed
|
||||
: DEFAULT_IMDS_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), getImdsTimeoutMs());
|
||||
|
||||
try {
|
||||
return await fetch(url, {
|
||||
...init,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchImdsToken(baseUrl: string): Promise<string | null> {
|
||||
const response = await fetchWithTimeout(`${baseUrl}/latest/api/token`, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
[IMDS_TOKEN_TTL_HEADER]: String(IMDS_TOKEN_TTL_SECONDS),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const token = await response.text();
|
||||
return token.trim() || null;
|
||||
}
|
||||
|
||||
async function fetchImdsText(
|
||||
baseUrl: string,
|
||||
token: string,
|
||||
path: string,
|
||||
): Promise<string | null> {
|
||||
const response = await fetchWithTimeout(`${baseUrl}${path}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[IMDS_TOKEN_HEADER]: token,
|
||||
},
|
||||
});
|
||||
|
||||
if (response.status === 404) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function fetchInfisicalCredentialsFromImds(): Promise<
|
||||
Partial<Pick<InfisicalCredentials, "clientId" | "clientSecret">>
|
||||
> {
|
||||
for (const baseUrl of getImdsBaseUrls()) {
|
||||
try {
|
||||
const token = await fetchImdsToken(baseUrl);
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const [clientId, clientSecret] = await Promise.all([
|
||||
fetchImdsText(
|
||||
baseUrl,
|
||||
token,
|
||||
`/latest/meta-data/secrets/${encodeURIComponent(INFISICAL_CLIENT_ID_KEY)}`,
|
||||
),
|
||||
fetchImdsText(
|
||||
baseUrl,
|
||||
token,
|
||||
`/latest/meta-data/secrets/${encodeURIComponent(INFISICAL_CLIENT_SECRET_KEY)}`,
|
||||
),
|
||||
]);
|
||||
|
||||
const trimmedClientId = clientId?.trim();
|
||||
const trimmedClientSecret = clientSecret?.trim();
|
||||
if (trimmedClientId || trimmedClientSecret) {
|
||||
const credentials: Partial<
|
||||
Pick<InfisicalCredentials, "clientId" | "clientSecret">
|
||||
> = {};
|
||||
if (trimmedClientId) {
|
||||
credentials.clientId = trimmedClientId;
|
||||
}
|
||||
if (trimmedClientSecret) {
|
||||
credentials.clientSecret = trimmedClientSecret;
|
||||
}
|
||||
return credentials;
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
async function getInfisicalCredentials(): Promise<InfisicalCredentials | null> {
|
||||
const projectId = process.env["INFISICAL_PROJECT_ID"]?.trim() ?? "";
|
||||
if (!projectId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const envClientId = process.env[INFISICAL_CLIENT_ID_KEY]?.trim();
|
||||
const envClientSecret = process.env[INFISICAL_CLIENT_SECRET_KEY]?.trim();
|
||||
const imdsCredentials: Partial<
|
||||
Pick<InfisicalCredentials, "clientId" | "clientSecret">
|
||||
> =
|
||||
envClientId && envClientSecret
|
||||
? {}
|
||||
: await fetchInfisicalCredentialsFromImds();
|
||||
const clientId = envClientId || imdsCredentials.clientId || "";
|
||||
const clientSecret = envClientSecret || imdsCredentials.clientSecret || "";
|
||||
|
||||
if (!clientId || !clientSecret) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { clientId, clientSecret, projectId };
|
||||
}
|
||||
|
||||
export function getSecret(key: KnownSecretKey): string | undefined {
|
||||
return secrets?.[key] ?? process.env[key];
|
||||
}
|
||||
@@ -43,11 +211,11 @@ export function getRawSecret(key: string): string | undefined {
|
||||
return rawSecrets?.[key] ?? process.env[key];
|
||||
}
|
||||
|
||||
const fetchSecrets = async (): Promise<{
|
||||
const fetchSecretsFromInfisical = async (): Promise<{
|
||||
known: SecretMap;
|
||||
raw: RawSecretMap;
|
||||
}> => {
|
||||
const credentials = getInfisicalCredentials();
|
||||
const credentials = await getInfisicalCredentials();
|
||||
if (!credentials) {
|
||||
return { known: {}, raw: {} };
|
||||
}
|
||||
@@ -79,6 +247,13 @@ const fetchSecrets = async (): Promise<{
|
||||
}
|
||||
};
|
||||
|
||||
const fetchSecrets = async (): Promise<{
|
||||
known: SecretMap;
|
||||
raw: RawSecretMap;
|
||||
}> => {
|
||||
return fetchSecretsFromInfisical();
|
||||
};
|
||||
|
||||
const loaded = await fetchSecrets();
|
||||
secrets = loaded.known;
|
||||
rawSecrets = loaded.raw;
|
||||
+112
@@ -29,6 +29,14 @@ function getRatioNumber(
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function getBoolean(envKey: string, fallback: boolean): boolean {
|
||||
const raw = process.env[envKey]?.trim().toLowerCase();
|
||||
if (!raw) return fallback;
|
||||
if (raw === "1" || raw === "true" || raw === "yes") return true;
|
||||
if (raw === "0" || raw === "false" || raw === "no") return false;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function getNodeEnv(): "dev" | "test" | "prod" {
|
||||
const value = process.env["NODE_ENV"]?.trim();
|
||||
return value === "test" || value === "prod" ? value : "dev";
|
||||
@@ -118,6 +126,94 @@ export function getEvalStreamMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
|
||||
}
|
||||
|
||||
export function getEvalAggTickMs(): number {
|
||||
return getPositiveNumber("EVAL_AGG_TICK_MS", 300_000);
|
||||
}
|
||||
|
||||
export function getEvalAggTtlSec(): number {
|
||||
return getPositiveNumber("EVAL_AGG_TTL_SEC", 604_800);
|
||||
}
|
||||
|
||||
export function getEvalAggLatencySampleCap(): number {
|
||||
return getPositiveNumber("EVAL_AGG_LATENCY_SAMPLE_CAP", 200);
|
||||
}
|
||||
|
||||
export function getEvalArchiveTickMs(): number {
|
||||
return getPositiveNumber("EVAL_ARCHIVE_TICK_MS", 3_600_000);
|
||||
}
|
||||
|
||||
export function getEvalArchiveTtlDays(): number {
|
||||
return getPositiveNumber("EVAL_ARCHIVE_TTL_DAYS", 30);
|
||||
}
|
||||
|
||||
export function getEvalArchiveBatchSize(): number {
|
||||
return getPositiveNumber("EVAL_ARCHIVE_BATCH_SIZE", 500);
|
||||
}
|
||||
|
||||
export function getEvalDriftTickMs(): number {
|
||||
return getPositiveNumber("EVAL_DRIFT_TICK_MS", 86_400_000);
|
||||
}
|
||||
|
||||
export function getEvalPromoteTickMs(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_TICK_MS", 86_400_000);
|
||||
}
|
||||
|
||||
export function getEvalPromoteLowScore(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_LOW_SCORE", 2);
|
||||
}
|
||||
|
||||
export function getEvalPromoteRareThreshold(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_RARE_THRESHOLD", 5);
|
||||
}
|
||||
|
||||
export function getEvalPromotePerCategoryLimit(): number {
|
||||
return getPositiveNumber("EVAL_PROMOTE_PER_CATEGORY_LIMIT", 10);
|
||||
}
|
||||
|
||||
export function getEvalReviewTickMs(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_TICK_MS", 300_000);
|
||||
}
|
||||
|
||||
export function getEvalReviewQueueMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_QUEUE_MAXLEN", 500);
|
||||
}
|
||||
|
||||
export function getEvalReviewLabelMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_LABEL_MAXLEN", 5_000);
|
||||
}
|
||||
|
||||
export function getEvalReviewPerCategoryLimit(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_PER_CATEGORY_LIMIT", 5);
|
||||
}
|
||||
|
||||
export function getEvalReviewDedupeTtlSec(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800);
|
||||
}
|
||||
|
||||
export function getEvalCalibrationMinCorr(): number {
|
||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||
}
|
||||
|
||||
export function getLlmProposeModel(): string {
|
||||
return process.env["LLM_PROPOSE_MODEL"]?.trim() || getLlmJudgeModel();
|
||||
}
|
||||
|
||||
export function getLlmProposeMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_PROPOSE_MAX_TOKENS", 1_024);
|
||||
}
|
||||
|
||||
export function getEvalTournamentMaxMatches(): number {
|
||||
return getPositiveNumber("EVAL_TOURNAMENT_MAX_MATCHES", 50);
|
||||
}
|
||||
|
||||
export function getEvalTournamentMinCases(): number {
|
||||
return getPositiveNumber("EVAL_TOURNAMENT_MIN_CASES", 10);
|
||||
}
|
||||
|
||||
export function getPromptVariantId(): string | undefined {
|
||||
return process.env["PROMPT_VARIANT_ID"]?.trim() || undefined;
|
||||
}
|
||||
|
||||
export function getMaxConversationMessages(): number {
|
||||
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
|
||||
}
|
||||
@@ -154,6 +250,18 @@ export function getLlmJudgeMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
|
||||
}
|
||||
|
||||
export function getEvalProcessLlmJudgeEnabled(): boolean {
|
||||
return getBoolean("EVAL_PROCESS_LLM_JUDGE_ENABLED", false);
|
||||
}
|
||||
|
||||
export function getEvalProcessLlmJudgeSamplingRate(): number {
|
||||
return getRatioNumber("EVAL_PROCESS_LLM_JUDGE_SAMPLING_RATE", 0.1);
|
||||
}
|
||||
|
||||
export function getEvalProcessLlmJudgeMaxTokens(): number {
|
||||
return getPositiveNumber("EVAL_PROCESS_LLM_JUDGE_MAX_TOKENS", 256);
|
||||
}
|
||||
|
||||
export function getSafetyMarginMs(): number {
|
||||
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
|
||||
}
|
||||
@@ -303,6 +411,10 @@ export function getAlertThresholdLlmErrorRate(): number {
|
||||
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
|
||||
}
|
||||
|
||||
export function getAlertThresholdDriftPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
|
||||
}
|
||||
|
||||
export function getAlertRateLimitSec(): number {
|
||||
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
aggregateRecentEvalResults,
|
||||
EVAL_AGG_CURSOR_KEY,
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
getSliceAggregateKey,
|
||||
} from "./aggregator.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly expires = new Map<string, number>();
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly strings = new Map<string, string>();
|
||||
readonly streams = new Map<string, StreamEntry[]>();
|
||||
|
||||
expireKey(key: string, ttlSeconds: number): Promise<boolean> {
|
||||
this.expires.set(key, ttlSeconds);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
_end: string,
|
||||
): Promise<StreamEntry[]> {
|
||||
const entries = this.streams.get(key) ?? [];
|
||||
if (!start.startsWith("(")) {
|
||||
return Promise.resolve(entries);
|
||||
}
|
||||
|
||||
const exclusiveStart = start.slice(1);
|
||||
return Promise.resolve(entries.filter((entry) => entry.id > exclusiveStart));
|
||||
}
|
||||
|
||||
scanKeys(pattern: string): Promise<string[]> {
|
||||
const prefix = pattern.replace(/\*$/u, "");
|
||||
return Promise.resolve(
|
||||
[...this.hashes.keys()].filter((key) => key.startsWith(prefix)),
|
||||
);
|
||||
}
|
||||
|
||||
setHash(key: string, fields: Record<string, string>): Promise<void> {
|
||||
this.hashes.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
function resultEntry(
|
||||
id: string,
|
||||
fields: Partial<Record<string, string>>,
|
||||
): StreamEntry {
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
type: "shadow-eval-result",
|
||||
tier: "0",
|
||||
intent: "status-question",
|
||||
latencyMs: "100",
|
||||
llmOverall: "4",
|
||||
validatorStatus: "pass",
|
||||
derived: JSON.stringify({
|
||||
tier: 0,
|
||||
intent: { kind: "status-question" },
|
||||
phase: { kind: "photo-pending" },
|
||||
}),
|
||||
...fields,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("aggregateRecentEvalResults", () => {
|
||||
it("rolls stream entries up into per-slice hashes", async () => {
|
||||
const garnet = new MemoryGarnet();
|
||||
garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [
|
||||
resultEntry("1-0", {}),
|
||||
resultEntry("2-0", {
|
||||
latencyMs: "300",
|
||||
llmOverall: "",
|
||||
llmJudge: JSON.stringify({
|
||||
skipped: false,
|
||||
scores: { overall: 2 },
|
||||
}),
|
||||
validatorStatus: "reject",
|
||||
}),
|
||||
resultEntry("3-0", {
|
||||
tier: "1",
|
||||
intent: "photo-question",
|
||||
latencyMs: "",
|
||||
llmOverall: "",
|
||||
validatorStatus: "soft-warn",
|
||||
derived: JSON.stringify({
|
||||
tier: 1,
|
||||
intent: { kind: "photo-question" },
|
||||
phase: { kind: "fresh" },
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
const result = await aggregateRecentEvalResults(garnet, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result).toEqual({ lastSeenId: "3-0", updatedSlices: 2 });
|
||||
expect(garnet.strings.get(EVAL_AGG_CURSOR_KEY)).toBe("3-0");
|
||||
|
||||
const key = getSliceAggregateKey({
|
||||
tier: 0,
|
||||
intent: "status-question",
|
||||
phase: "photo-pending",
|
||||
});
|
||||
expect(garnet.hashes.get(key)).toMatchObject({
|
||||
count: "2",
|
||||
llmOverallSum: "6",
|
||||
llmOverallCount: "2",
|
||||
validatorRejectCount: "1",
|
||||
latencySumMs: "400",
|
||||
});
|
||||
expect(JSON.parse(garnet.hashes.get(key)?.["latencySamples"] ?? "[]")).toEqual([
|
||||
100,
|
||||
300,
|
||||
]);
|
||||
expect(garnet.expires.get(key)).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,396 @@
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getEvalAggLatencySampleCap,
|
||||
getEvalAggTtlSec,
|
||||
} from "../config.js";
|
||||
|
||||
export const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
||||
export const EVAL_AGG_CURSOR_KEY = "lolympus-agent:eval:agg:cursor";
|
||||
const EVAL_AGG_SLICE_KEY_PREFIX = "lolympus-agent:eval:agg:";
|
||||
|
||||
export interface SliceKey {
|
||||
tier: 0 | 1 | 2;
|
||||
intent: string;
|
||||
phase: string;
|
||||
}
|
||||
|
||||
export interface SliceMetrics {
|
||||
count: number;
|
||||
llmOverallSum: number;
|
||||
llmOverallCount: number;
|
||||
validatorRejectCount: number;
|
||||
validatorSoftWarnCount: number;
|
||||
latencySumMs: number;
|
||||
latencySamples: number[];
|
||||
processJudgeSum: number;
|
||||
processJudgeCount: number;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface SliceReportRow {
|
||||
slice: SliceKey;
|
||||
count: number;
|
||||
llmOverallAvg: number | null;
|
||||
processJudgeAvg: number | null;
|
||||
validatorRejectRate: number;
|
||||
validatorSoftWarnRate: number;
|
||||
p50LatencyMs: number | null;
|
||||
p95LatencyMs: number | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
type AggregatorGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
| "expireKey"
|
||||
| "getHash"
|
||||
| "readStreamRange"
|
||||
| "scanKeys"
|
||||
| "setHash"
|
||||
| "setString"
|
||||
>;
|
||||
|
||||
interface EvalResultObservation {
|
||||
latencyMs: number | null;
|
||||
llmOverall: number | null;
|
||||
processJudgeScore: number | null;
|
||||
slice: SliceKey;
|
||||
validatorStatus: string | null;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function parseJsonRecord(value: string | undefined): Record<string, unknown> {
|
||||
if (!value?.trim()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
return asRecord(JSON.parse(value));
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function parseFiniteNumber(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
||||
const parsed = parseFiniteNumber(value);
|
||||
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
||||
return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getNestedString(
|
||||
record: Record<string, unknown>,
|
||||
first: string,
|
||||
second: string,
|
||||
): string | null {
|
||||
const nested = asRecord(record[first]);
|
||||
const value = nested[second];
|
||||
return typeof value === "string" && value.trim() ? value : null;
|
||||
}
|
||||
|
||||
function extractLlmOverall(message: Record<string, string>): number | null {
|
||||
const direct = parseFiniteNumber(message["llmOverall"]);
|
||||
if (direct !== null) {
|
||||
return direct;
|
||||
}
|
||||
|
||||
const llmJudge = parseJsonRecord(message["llmJudge"]);
|
||||
const scores = asRecord(llmJudge["scores"]);
|
||||
return parseFiniteNumber(scores["overall"]);
|
||||
}
|
||||
|
||||
function extractProcessJudgeScore(
|
||||
message: Record<string, string>,
|
||||
): number | null {
|
||||
const processJudge = parseJsonRecord(message["processJudge"]);
|
||||
return parseFiniteNumber(processJudge["score"]);
|
||||
}
|
||||
|
||||
function extractValidatorStatus(message: Record<string, string>): string | null {
|
||||
if (message["validatorStatus"]?.trim()) {
|
||||
return message["validatorStatus"] ?? null;
|
||||
}
|
||||
|
||||
const validation = parseJsonRecord(message["validation"]);
|
||||
const status = validation["status"];
|
||||
return typeof status === "string" && status.trim() ? status : null;
|
||||
}
|
||||
|
||||
function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
||||
const derived = parseJsonRecord(entry.message["derived"]);
|
||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
||||
const intent =
|
||||
entry.message["intent"]?.trim() ||
|
||||
getNestedString(derived, "intent", "kind");
|
||||
const phase =
|
||||
entry.message["phase"]?.trim() || getNestedString(derived, "phase", "kind");
|
||||
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
||||
llmOverall: extractLlmOverall(entry.message),
|
||||
processJudgeScore: extractProcessJudgeScore(entry.message),
|
||||
slice: { tier, intent, phase },
|
||||
validatorStatus: extractValidatorStatus(entry.message),
|
||||
};
|
||||
}
|
||||
|
||||
function parseNumberField(
|
||||
hash: Record<string, string>,
|
||||
field: keyof SliceMetrics,
|
||||
): number {
|
||||
const value = parseFiniteNumber(hash[field]);
|
||||
return value ?? 0;
|
||||
}
|
||||
|
||||
function parseLatencySamples(hash: Record<string, string>): number[] {
|
||||
const raw = hash["latencySamples"];
|
||||
if (!raw?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return [];
|
||||
}
|
||||
return parsed.flatMap((item) => {
|
||||
const value = parseFiniteNumber(item);
|
||||
return value === null ? [] : [value];
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function metricsFromHash(hash: Record<string, string>): SliceMetrics {
|
||||
return {
|
||||
count: parseNumberField(hash, "count"),
|
||||
llmOverallSum: parseNumberField(hash, "llmOverallSum"),
|
||||
llmOverallCount: parseNumberField(hash, "llmOverallCount"),
|
||||
validatorRejectCount: parseNumberField(hash, "validatorRejectCount"),
|
||||
validatorSoftWarnCount: parseNumberField(hash, "validatorSoftWarnCount"),
|
||||
latencySumMs: parseNumberField(hash, "latencySumMs"),
|
||||
latencySamples: parseLatencySamples(hash),
|
||||
processJudgeSum: parseNumberField(hash, "processJudgeSum"),
|
||||
processJudgeCount: parseNumberField(hash, "processJudgeCount"),
|
||||
updatedAt: hash["updatedAt"] ?? new Date(0).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function metricsToHash(metrics: SliceMetrics): Record<string, string> {
|
||||
return {
|
||||
count: String(metrics.count),
|
||||
llmOverallSum: String(metrics.llmOverallSum),
|
||||
llmOverallCount: String(metrics.llmOverallCount),
|
||||
validatorRejectCount: String(metrics.validatorRejectCount),
|
||||
validatorSoftWarnCount: String(metrics.validatorSoftWarnCount),
|
||||
latencySumMs: String(metrics.latencySumMs),
|
||||
latencySamples: JSON.stringify(metrics.latencySamples),
|
||||
processJudgeSum: String(metrics.processJudgeSum),
|
||||
processJudgeCount: String(metrics.processJudgeCount),
|
||||
updatedAt: metrics.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function updateMetrics(
|
||||
current: SliceMetrics,
|
||||
observation: EvalResultObservation,
|
||||
nowMs: number,
|
||||
): SliceMetrics {
|
||||
const latencySamples = [...current.latencySamples];
|
||||
const latencySampleCap = getEvalAggLatencySampleCap();
|
||||
|
||||
if (observation.latencyMs !== null) {
|
||||
latencySamples.push(observation.latencyMs);
|
||||
if (latencySamples.length > latencySampleCap) {
|
||||
latencySamples.splice(0, latencySamples.length - latencySampleCap);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
count: current.count + 1,
|
||||
llmOverallSum:
|
||||
current.llmOverallSum + (observation.llmOverall ?? 0),
|
||||
llmOverallCount:
|
||||
current.llmOverallCount + (observation.llmOverall === null ? 0 : 1),
|
||||
validatorRejectCount:
|
||||
current.validatorRejectCount +
|
||||
(observation.validatorStatus === "reject" ? 1 : 0),
|
||||
validatorSoftWarnCount:
|
||||
current.validatorSoftWarnCount +
|
||||
(observation.validatorStatus === "soft-warn" ? 1 : 0),
|
||||
latencySumMs: current.latencySumMs + (observation.latencyMs ?? 0),
|
||||
latencySamples,
|
||||
processJudgeSum:
|
||||
current.processJudgeSum + (observation.processJudgeScore ?? 0),
|
||||
processJudgeCount:
|
||||
current.processJudgeCount +
|
||||
(observation.processJudgeScore === null ? 0 : 1),
|
||||
updatedAt: new Date(nowMs).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function calculatePercentile(values: number[], percentile: number): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : round(selected);
|
||||
}
|
||||
|
||||
function average(sum: number, count: number): number | null {
|
||||
return count > 0 ? round(sum / count) : null;
|
||||
}
|
||||
|
||||
function sliceKeyFromString(value: string): SliceKey | null {
|
||||
const [tierRaw, intent, phase] = value.split(":");
|
||||
const tier = parseTier(tierRaw);
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { tier, intent, phase };
|
||||
}
|
||||
|
||||
function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
||||
if (!key.startsWith(EVAL_AGG_SLICE_KEY_PREFIX)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return sliceKeyFromString(key.slice(EVAL_AGG_SLICE_KEY_PREFIX.length));
|
||||
}
|
||||
|
||||
export function sliceKeyToString(key: SliceKey): string {
|
||||
return `${key.tier}:${key.intent}:${key.phase}`;
|
||||
}
|
||||
|
||||
export function getSliceAggregateKey(key: SliceKey): string {
|
||||
return `${EVAL_AGG_SLICE_KEY_PREFIX}${sliceKeyToString(key)}`;
|
||||
}
|
||||
|
||||
export function sliceMetricsToReportRow(
|
||||
slice: SliceKey,
|
||||
metrics: SliceMetrics,
|
||||
): SliceReportRow {
|
||||
return {
|
||||
slice,
|
||||
count: metrics.count,
|
||||
llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount),
|
||||
processJudgeAvg: average(
|
||||
metrics.processJudgeSum,
|
||||
metrics.processJudgeCount,
|
||||
),
|
||||
validatorRejectRate:
|
||||
metrics.count === 0
|
||||
? 0
|
||||
: round(metrics.validatorRejectCount / metrics.count),
|
||||
validatorSoftWarnRate:
|
||||
metrics.count === 0
|
||||
? 0
|
||||
: round(metrics.validatorSoftWarnCount / metrics.count),
|
||||
p50LatencyMs: calculatePercentile(metrics.latencySamples, 0.5),
|
||||
p95LatencyMs: calculatePercentile(metrics.latencySamples, 0.95),
|
||||
updatedAt: metrics.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export async function aggregateRecentEvalResults(
|
||||
garnetClient: AggregatorGarnetClient,
|
||||
options: { sinceId: string; nowMs: number },
|
||||
): Promise<{ lastSeenId: string; updatedSlices: number }> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
lastSeenId: options.sinceId,
|
||||
updatedSlices: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const updatedKeys = new Set<string>();
|
||||
let lastSeenId = options.sinceId;
|
||||
|
||||
for (const entry of entries) {
|
||||
lastSeenId = entry.id;
|
||||
const observation = parseObservation(entry);
|
||||
if (!observation) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = getSliceAggregateKey(observation.slice);
|
||||
const current = metricsFromHash(await garnetClient.getHash(key));
|
||||
const next = updateMetrics(current, observation, options.nowMs);
|
||||
await garnetClient.setHash(key, metricsToHash(next));
|
||||
await garnetClient.expireKey(key, getEvalAggTtlSec());
|
||||
updatedKeys.add(key);
|
||||
}
|
||||
|
||||
await garnetClient.setString(EVAL_AGG_CURSOR_KEY, lastSeenId);
|
||||
|
||||
return {
|
||||
lastSeenId,
|
||||
updatedSlices: updatedKeys.size,
|
||||
};
|
||||
}
|
||||
|
||||
export async function readAggregatedSliceReportRows(
|
||||
garnetClient: AggregatorGarnetClient,
|
||||
): Promise<SliceReportRow[]> {
|
||||
const keys = await garnetClient.scanKeys(`${EVAL_AGG_SLICE_KEY_PREFIX}*`);
|
||||
const rows: SliceReportRow[] = [];
|
||||
|
||||
for (const key of keys) {
|
||||
const slice = sliceKeyFromRedisKey(key);
|
||||
if (!slice) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metrics = metricsFromHash(await garnetClient.getHash(key));
|
||||
rows.push(sliceMetricsToReportRow(slice, metrics));
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
|
||||
const EVAL_ANON_SALT = "lolympus-agent-v1" as const;
|
||||
|
||||
@@ -51,3 +52,32 @@ export function anonymizeRelativeTimestamp(
|
||||
const seconds = Math.round((timestamp - baseTimestamp.getTime()) / 1000);
|
||||
return `${seconds >= 0 ? "+" : ""}${seconds}s`;
|
||||
}
|
||||
|
||||
function capText(value: string, maxLength = 600): string {
|
||||
return value.length > maxLength ? value.slice(0, maxLength) : value;
|
||||
}
|
||||
|
||||
function anonymizeUnknown(value: unknown): unknown {
|
||||
if (typeof value === "string") {
|
||||
return capText(anonymizeText(value));
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => anonymizeUnknown(item));
|
||||
}
|
||||
if (typeof value === "object" && value !== null) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, nested]) => [
|
||||
key,
|
||||
anonymizeUnknown(nested),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
export function anonymizeProcessTrace(
|
||||
trace: ProcessTraceSnapshot,
|
||||
): ProcessTraceSnapshot {
|
||||
return anonymizeUnknown(trace) as ProcessTraceSnapshot;
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
archiveEvalResults,
|
||||
EVAL_ARCHIVE_CURSOR_KEY,
|
||||
EVAL_ARCHIVE_COLLECTION,
|
||||
type EvalArchiveEntry,
|
||||
} from "./archive.js";
|
||||
import { EVAL_RESULTS_STREAM_KEY } from "./aggregator.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly strings = new Map<string, string>();
|
||||
|
||||
constructor(private readonly entries: StreamEntry[]) {}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
_end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(EVAL_RESULTS_STREAM_KEY);
|
||||
const filtered = start.startsWith("(")
|
||||
? this.entries.filter((entry) => entry.id > start.slice(1))
|
||||
: this.entries;
|
||||
return Promise.resolve(filtered.slice(0, options?.count));
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryCollection {
|
||||
readonly docs = new Map<string, EvalArchiveEntry>();
|
||||
readonly indexes: Array<Record<string, unknown>> = [];
|
||||
|
||||
bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> {
|
||||
let upsertedCount = 0;
|
||||
for (const operation of operations) {
|
||||
const updateOne = (operation as {
|
||||
updateOne: {
|
||||
filter: { streamId: string };
|
||||
update: { $setOnInsert: EvalArchiveEntry };
|
||||
};
|
||||
}).updateOne;
|
||||
const streamId = updateOne.filter.streamId;
|
||||
if (!this.docs.has(streamId)) {
|
||||
this.docs.set(streamId, updateOne.update.$setOnInsert);
|
||||
upsertedCount += 1;
|
||||
}
|
||||
}
|
||||
return Promise.resolve({ upsertedCount });
|
||||
}
|
||||
|
||||
createIndex(keys: Record<string, unknown>): Promise<string> {
|
||||
this.indexes.push(keys);
|
||||
return Promise.resolve("idx");
|
||||
}
|
||||
|
||||
find(): never {
|
||||
throw new Error("not used");
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryConnection {
|
||||
readonly collectionInstance = new MemoryCollection();
|
||||
|
||||
collection(name: string): MemoryCollection {
|
||||
expect(name).toBe(EVAL_ARCHIVE_COLLECTION);
|
||||
return this.collectionInstance;
|
||||
}
|
||||
}
|
||||
|
||||
function streamEntry(id: string): StreamEntry {
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
input: JSON.stringify({
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "상태 알려줘",
|
||||
},
|
||||
}),
|
||||
output: JSON.stringify({
|
||||
requestId: "req-1",
|
||||
text: "사진 인증만 남았어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
}),
|
||||
derived: JSON.stringify({
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
}),
|
||||
classified: JSON.stringify({
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
}),
|
||||
validation: JSON.stringify({
|
||||
status: "pass",
|
||||
text: "사진 인증만 남았어요.",
|
||||
reasons: [],
|
||||
}),
|
||||
llmJudge: JSON.stringify({
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: 4,
|
||||
accuracy: 5,
|
||||
concision: 4,
|
||||
scope: 5,
|
||||
overall: 4.5,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
}),
|
||||
tier: "0",
|
||||
latencyMs: "123",
|
||||
evaluatedAt: "2026-05-01T00:00:00.000Z",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("archiveEvalResults", () => {
|
||||
it("archives stream entries idempotently and advances the cursor", async () => {
|
||||
const garnet = new MemoryGarnet([streamEntry("1-0")]);
|
||||
const connection = new MemoryConnection();
|
||||
|
||||
const first = await archiveEvalResults(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
const second = await archiveEvalResults(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(first).toEqual({ lastArchivedId: "1-0", archivedCount: 1 });
|
||||
expect(second).toEqual({ lastArchivedId: "1-0", archivedCount: 0 });
|
||||
expect(garnet.strings.get(EVAL_ARCHIVE_CURSOR_KEY)).toBe("1-0");
|
||||
expect(connection.collectionInstance.docs.get("1-0")).toMatchObject({
|
||||
streamId: "1-0",
|
||||
latencyMs: 123,
|
||||
tier: 0,
|
||||
});
|
||||
expect(connection.collectionInstance.indexes).toContainEqual({ streamId: 1 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,404 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ValidationResult } from "@skills/onboarding/runtime/validator";
|
||||
import {
|
||||
getEvalArchiveBatchSize,
|
||||
getEvalArchiveTtlDays,
|
||||
} from "../config.js";
|
||||
import type { LlmJudgeResult } from "./types.js";
|
||||
import type { ProcessJudgeResult } from "./types.js";
|
||||
import {
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
type SliceKey,
|
||||
type SliceReportRow,
|
||||
} from "./aggregator.js";
|
||||
|
||||
export const EVAL_ARCHIVE_CURSOR_KEY = "lolympus-agent:eval:archive:cursor";
|
||||
export const EVAL_ARCHIVE_COLLECTION = "eval_archive";
|
||||
|
||||
export interface EvalArchiveEntry {
|
||||
streamId: string;
|
||||
evaluatedAt: Date;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
derived: DerivedState;
|
||||
classified: ClassifiedIntent;
|
||||
validation: ValidationResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
process?: unknown;
|
||||
latencyMs: number | null;
|
||||
tier: 0 | 1 | 2;
|
||||
}
|
||||
|
||||
interface BulkWriteResultLike {
|
||||
upsertedCount?: number | undefined;
|
||||
}
|
||||
|
||||
interface FindCursorLike<T> {
|
||||
limit(value: number): FindCursorLike<T>;
|
||||
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
|
||||
toArray(): Promise<T[]>;
|
||||
}
|
||||
|
||||
interface EvalArchiveCollectionLike {
|
||||
bulkWrite(
|
||||
operations: unknown[],
|
||||
options?: { ordered?: boolean | undefined },
|
||||
): Promise<BulkWriteResultLike>;
|
||||
createIndex(
|
||||
keys: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
): Promise<unknown>;
|
||||
find(filter: Record<string, unknown>): FindCursorLike<EvalArchiveEntry>;
|
||||
}
|
||||
|
||||
export interface MongoConnectionLike {
|
||||
collection(name: string): unknown;
|
||||
}
|
||||
|
||||
type ArchiveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"readStreamRange" | "setString"
|
||||
>;
|
||||
|
||||
function parseJsonField<T>(value: string | undefined, field: string): T {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`Missing ${field} field`);
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseFiniteNumber(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function parseTier(value: unknown): 0 | 1 | 2 | null {
|
||||
const parsed = parseFiniteNumber(value);
|
||||
if (parsed === 0 || parsed === 1 || parsed === 2) {
|
||||
return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseEvaluatedAt(value: string | undefined, nowMs: number): Date {
|
||||
if (!value?.trim()) {
|
||||
return new Date(nowMs);
|
||||
}
|
||||
|
||||
const parsed = new Date(value);
|
||||
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
|
||||
}
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function calculatePercentile(values: number[], percentile: number): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : round(selected);
|
||||
}
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
return {
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
};
|
||||
}
|
||||
|
||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
if (entry.llmJudge.skipped) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function sliceKeyToMapKey(slice: SliceKey): string {
|
||||
return `${slice.tier}:${slice.intent}:${slice.phase}`;
|
||||
}
|
||||
|
||||
function parseEvalArchiveEntry(
|
||||
entry: StreamEntry,
|
||||
nowMs: number,
|
||||
): EvalArchiveEntry {
|
||||
const input = parseJsonField<{ request: AgentRequest }>(
|
||||
entry.message["input"],
|
||||
"input",
|
||||
);
|
||||
const derived = parseJsonField<DerivedState>(
|
||||
entry.message["derived"],
|
||||
"derived",
|
||||
);
|
||||
const processJudge = parseOptionalJsonField<ProcessJudgeResult>(
|
||||
entry.message["processJudge"],
|
||||
);
|
||||
const process = parseOptionalJsonField(entry.message["process"]);
|
||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived.tier);
|
||||
|
||||
if (tier === null) {
|
||||
throw new Error("Missing valid tier");
|
||||
}
|
||||
|
||||
const archiveEntry: EvalArchiveEntry = {
|
||||
streamId: entry.id,
|
||||
evaluatedAt: parseEvaluatedAt(entry.message["evaluatedAt"], nowMs),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(entry.message["output"], "output"),
|
||||
derived,
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
entry.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
validation: parseJsonField<ValidationResult>(
|
||||
entry.message["validation"],
|
||||
"validation",
|
||||
),
|
||||
llmJudge: parseJsonField<LlmJudgeResult>(
|
||||
entry.message["llmJudge"],
|
||||
"llmJudge",
|
||||
),
|
||||
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
||||
tier,
|
||||
};
|
||||
|
||||
if (processJudge !== undefined) {
|
||||
archiveEntry.processJudge = processJudge;
|
||||
}
|
||||
if (process !== undefined) {
|
||||
archiveEntry.process = process;
|
||||
}
|
||||
|
||||
return archiveEntry;
|
||||
}
|
||||
|
||||
async function ensureEvalArchiveIndexes(
|
||||
collection: EvalArchiveCollectionLike,
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ streamId: 1 }, { unique: true }),
|
||||
collection.createIndex(
|
||||
{ evaluatedAt: 1 },
|
||||
{ expireAfterSeconds: getEvalArchiveTtlDays() * 24 * 60 * 60 },
|
||||
),
|
||||
collection.createIndex({
|
||||
tier: 1,
|
||||
"derived.intent.kind": 1,
|
||||
"derived.phase.kind": 1,
|
||||
evaluatedAt: -1,
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
export function getEvalArchiveCollection(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
): EvalArchiveCollectionLike {
|
||||
return mongoConnection.collection(EVAL_ARCHIVE_COLLECTION) as EvalArchiveCollectionLike;
|
||||
}
|
||||
|
||||
export async function archiveEvalResults(
|
||||
garnetClient: ArchiveGarnetClient,
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options: {
|
||||
batchSize?: number | undefined;
|
||||
nowMs: number;
|
||||
sinceId: string;
|
||||
},
|
||||
): Promise<{ lastArchivedId: string; archivedCount: number }> {
|
||||
const collection = getEvalArchiveCollection(mongoConnection);
|
||||
await ensureEvalArchiveIndexes(collection);
|
||||
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
{
|
||||
count: options.batchSize ?? getEvalArchiveBatchSize(),
|
||||
},
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
lastArchivedId: options.sinceId,
|
||||
archivedCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
|
||||
const archiveEntries = entries.flatMap((entry) => {
|
||||
try {
|
||||
return [parseEvalArchiveEntry(entry, options.nowMs)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
let archivedCount = 0;
|
||||
if (archiveEntries.length > 0) {
|
||||
const result = await collection.bulkWrite(
|
||||
archiveEntries.map((entry) => ({
|
||||
updateOne: {
|
||||
filter: { streamId: entry.streamId },
|
||||
update: { $setOnInsert: entry },
|
||||
upsert: true,
|
||||
},
|
||||
})),
|
||||
{ ordered: false },
|
||||
);
|
||||
archivedCount = result.upsertedCount ?? 0;
|
||||
}
|
||||
|
||||
await garnetClient.setString(EVAL_ARCHIVE_CURSOR_KEY, lastArchivedId);
|
||||
|
||||
return {
|
||||
lastArchivedId,
|
||||
archivedCount,
|
||||
};
|
||||
}
|
||||
|
||||
export function archiveEntryToSliceReportRows(
|
||||
entries: EvalArchiveEntry[],
|
||||
): SliceReportRow[] {
|
||||
const grouped = new Map<
|
||||
string,
|
||||
{
|
||||
count: number;
|
||||
latencySamples: number[];
|
||||
llmOverallCount: number;
|
||||
llmOverallSum: number;
|
||||
slice: SliceKey;
|
||||
validatorRejectCount: number;
|
||||
validatorSoftWarnCount: number;
|
||||
updatedAt: Date;
|
||||
}
|
||||
>();
|
||||
|
||||
for (const entry of entries) {
|
||||
const slice = getSlice(entry);
|
||||
const key = sliceKeyToMapKey(slice);
|
||||
const current =
|
||||
grouped.get(key) ??
|
||||
{
|
||||
count: 0,
|
||||
latencySamples: [],
|
||||
llmOverallCount: 0,
|
||||
llmOverallSum: 0,
|
||||
slice,
|
||||
validatorRejectCount: 0,
|
||||
validatorSoftWarnCount: 0,
|
||||
updatedAt: new Date(0),
|
||||
};
|
||||
|
||||
current.count += 1;
|
||||
if (entry.latencyMs !== null) {
|
||||
current.latencySamples.push(entry.latencyMs);
|
||||
}
|
||||
const llmOverall = getLlmOverall(entry);
|
||||
if (llmOverall !== null) {
|
||||
current.llmOverallCount += 1;
|
||||
current.llmOverallSum += llmOverall;
|
||||
}
|
||||
if (entry.validation.status === "reject") {
|
||||
current.validatorRejectCount += 1;
|
||||
}
|
||||
if (entry.validation.status === "soft-warn") {
|
||||
current.validatorSoftWarnCount += 1;
|
||||
}
|
||||
if (entry.evaluatedAt > current.updatedAt) {
|
||||
current.updatedAt = entry.evaluatedAt;
|
||||
}
|
||||
|
||||
grouped.set(key, current);
|
||||
}
|
||||
|
||||
return [...grouped.values()].map((group) => ({
|
||||
slice: group.slice,
|
||||
count: group.count,
|
||||
llmOverallAvg:
|
||||
group.llmOverallCount === 0
|
||||
? null
|
||||
: round(group.llmOverallSum / group.llmOverallCount),
|
||||
processJudgeAvg: null,
|
||||
validatorRejectRate:
|
||||
group.count === 0 ? 0 : round(group.validatorRejectCount / group.count),
|
||||
validatorSoftWarnRate:
|
||||
group.count === 0
|
||||
? 0
|
||||
: round(group.validatorSoftWarnCount / group.count),
|
||||
p50LatencyMs: calculatePercentile(group.latencySamples, 0.5),
|
||||
p95LatencyMs: calculatePercentile(group.latencySamples, 0.95),
|
||||
updatedAt: group.updatedAt.toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function queryArchiveSliceReportRows(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
sinceMs?: number | undefined;
|
||||
sort?: "count-desc" | "score-asc" | "score-desc" | undefined;
|
||||
},
|
||||
): Promise<SliceReportRow[]> {
|
||||
const sinceMs = options?.sinceMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||
const since = new Date(Date.now() - sinceMs);
|
||||
const archiveEntries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({ evaluatedAt: { $gte: since } })
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(10_000)
|
||||
.toArray();
|
||||
const rows = archiveEntryToSliceReportRows(archiveEntries);
|
||||
const sort = options?.sort ?? "count-desc";
|
||||
|
||||
rows.sort((left, right) => {
|
||||
if (sort === "score-asc") {
|
||||
return (left.llmOverallAvg ?? Number.POSITIVE_INFINITY) -
|
||||
(right.llmOverallAvg ?? Number.POSITIVE_INFINITY);
|
||||
}
|
||||
if (sort === "score-desc") {
|
||||
return (right.llmOverallAvg ?? Number.NEGATIVE_INFINITY) -
|
||||
(left.llmOverallAvg ?? Number.NEGATIVE_INFINITY);
|
||||
}
|
||||
return right.count - left.count;
|
||||
});
|
||||
|
||||
return rows.slice(0, options?.limit ?? 20);
|
||||
}
|
||||
+3
-1
@@ -50,7 +50,9 @@ export async function captureGoldenSetScaffold(
|
||||
process.env["MONGODB_CONNECTION_STRING"];
|
||||
|
||||
if (!connectionString) {
|
||||
throw new Error("Missing MONGODB_CONNECTION_STRING for eval capture");
|
||||
throw new Error(
|
||||
"Missing MONGODB_CONNECTION_STRING for eval capture. Ensure IMDS exposes Infisical client credentials and Infisical contains MONGODB_CONNECTION_STRING, or set it directly in the environment.",
|
||||
);
|
||||
}
|
||||
|
||||
await initMongoDB(connectionString);
|
||||
|
||||
+397
@@ -1,3 +1,7 @@
|
||||
import type { MongoConnectionLike } from "./archive.js";
|
||||
import type { SliceKey, SliceReportRow } from "./aggregator.js";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
@@ -7,6 +11,78 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseDurationMs(value: string | undefined): number {
|
||||
const raw = value?.trim().toLowerCase() || "7d";
|
||||
const match = raw.match(/^(\d+)(h|d)$/u);
|
||||
if (!match) {
|
||||
throw new Error("--since must use a duration like 24h, 7d, or 30d");
|
||||
}
|
||||
|
||||
const amount = Number(match[1]);
|
||||
const unit = match[2];
|
||||
if (!Number.isInteger(amount) || amount <= 0) {
|
||||
throw new Error("--since must be a positive duration");
|
||||
}
|
||||
|
||||
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function parseCsv(value: string | undefined): string[] {
|
||||
if (!value?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function parseSlice(value: string | undefined): SliceKey {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(
|
||||
"--slice must use tier=0,intent=status-question,phase=photo-pending",
|
||||
);
|
||||
}
|
||||
|
||||
const fields: Record<string, string> = {};
|
||||
for (const pair of value.split(",")) {
|
||||
const [key, ...rest] = pair.split("=");
|
||||
const normalizedKey = key?.trim();
|
||||
if (normalizedKey) {
|
||||
fields[normalizedKey] = rest.join("=").trim();
|
||||
}
|
||||
}
|
||||
const tier = Number(fields["tier"]);
|
||||
const intent = fields["intent"];
|
||||
const phase = fields["phase"];
|
||||
|
||||
if (
|
||||
(tier !== 0 && tier !== 1 && tier !== 2) ||
|
||||
!intent?.trim() ||
|
||||
!phase?.trim()
|
||||
) {
|
||||
throw new Error(
|
||||
"--slice must include tier=0|1|2,intent=<kind>,phase=<kind>",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
};
|
||||
}
|
||||
|
||||
interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
@@ -108,6 +184,122 @@ function printRunSummary(report: {
|
||||
console.info(`[eval] report=${reportPath}`);
|
||||
}
|
||||
|
||||
function formatNullable(value: number | null): string {
|
||||
return value === null ? "n/a" : String(value);
|
||||
}
|
||||
|
||||
function printSliceRows(rows: SliceReportRow[]): void {
|
||||
console.info(
|
||||
[
|
||||
"tier",
|
||||
"intent",
|
||||
"phase",
|
||||
"count",
|
||||
"llmOverallAvg",
|
||||
"validatorRejectRate",
|
||||
"p50LatencyMs",
|
||||
"p95LatencyMs",
|
||||
"updatedAt",
|
||||
].join("\t"),
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
console.info(
|
||||
[
|
||||
row.slice.tier,
|
||||
row.slice.intent,
|
||||
row.slice.phase,
|
||||
row.count,
|
||||
formatNullable(row.llmOverallAvg),
|
||||
row.validatorRejectRate,
|
||||
formatNullable(row.p50LatencyMs),
|
||||
formatNullable(row.p95LatencyMs),
|
||||
row.updatedAt,
|
||||
].join("\t"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function printCalibrationReport(report: {
|
||||
generatedAt: string;
|
||||
meanGapOverall: number;
|
||||
pairs: number;
|
||||
pearsonByDim: {
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
friendliness: number;
|
||||
scope: number;
|
||||
};
|
||||
pearsonOverall: number;
|
||||
worstDisagreements: Array<{
|
||||
evalStreamId: string;
|
||||
humanScore: number;
|
||||
llmOverall: number;
|
||||
}>;
|
||||
}): void {
|
||||
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
||||
console.info(`[eval] pairs=${report.pairs}`);
|
||||
console.info(`[eval] pearsonOverall=${report.pearsonOverall}`);
|
||||
console.info(`[eval] meanGapOverall=${report.meanGapOverall}`);
|
||||
console.info(
|
||||
`[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`,
|
||||
);
|
||||
for (const item of report.worstDisagreements) {
|
||||
console.info(
|
||||
`[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function printTournamentResult(result: {
|
||||
generatedAt: string;
|
||||
matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>;
|
||||
variants: Array<{ elo: number; id: string; matches: number; wins: number }>;
|
||||
}): void {
|
||||
console.info(`[eval] generatedAt=${result.generatedAt}`);
|
||||
console.info(`[eval] matches=${result.matches.length}`);
|
||||
for (const variant of result.variants) {
|
||||
console.info(
|
||||
`[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function connectMongoForEval(): Promise<{
|
||||
close: () => Promise<void>;
|
||||
connection: MongoConnectionLike;
|
||||
}> {
|
||||
const { initMongoDB, closeMongoDB } = await import("@common/mongodb");
|
||||
const { getSecret } = await import("@common/secret-manager");
|
||||
const connectionString =
|
||||
getSecret("MONGODB_CONNECTION_STRING") ??
|
||||
process.env["MONGODB_CONNECTION_STRING"];
|
||||
|
||||
if (!connectionString) {
|
||||
throw new Error(
|
||||
"Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
connection: await initMongoDB(connectionString),
|
||||
close: closeMongoDB,
|
||||
};
|
||||
}
|
||||
|
||||
async function connectGarnetForEval(): Promise<{
|
||||
close: () => Promise<void>;
|
||||
garnetClient: GarnetClient;
|
||||
}> {
|
||||
const { GarnetClient } = await import("@common/garnet");
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
return {
|
||||
garnetClient,
|
||||
close: () => garnetClient.close(),
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
||||
loadEnvironmentFromNodeEnv();
|
||||
@@ -174,6 +366,211 @@ async function main(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "report") {
|
||||
const { queryArchiveSliceReportRows } = await import("./archive.js");
|
||||
const limit =
|
||||
parsePositiveInteger(getFlagString(flags, "worst")) ??
|
||||
parsePositiveInteger(getFlagString(flags, "top")) ??
|
||||
20;
|
||||
const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc";
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const rows = await queryArchiveSliceReportRows(mongo.connection, {
|
||||
limit,
|
||||
sinceMs: parseDurationMs(getFlagString(flags, "since")),
|
||||
sort,
|
||||
});
|
||||
printSliceRows(rows);
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "promote") {
|
||||
const { appendCandidatesToJsonl, selectPromotionCandidates } = await import(
|
||||
"./promote.js"
|
||||
);
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const candidates = await selectPromotionCandidates(mongo.connection, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")),
|
||||
});
|
||||
|
||||
if (hasFlag(flags, "dry-run")) {
|
||||
for (const candidate of candidates) {
|
||||
const overall = candidate.llmJudge.skipped
|
||||
? "n/a"
|
||||
: candidate.llmJudge.scores.overall;
|
||||
console.info(
|
||||
`[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`,
|
||||
);
|
||||
}
|
||||
console.info(`[eval] dryRunCandidates=${candidates.length}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await appendCandidatesToJsonl(
|
||||
candidates,
|
||||
getFlagString(flags, "output"),
|
||||
);
|
||||
console.info(
|
||||
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
|
||||
);
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "review") {
|
||||
const { parseReviewReason, runReviewCli } = await import("./review/cli.js");
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
const result = await runReviewCli(garnet.garnetClient, {
|
||||
exportLabeled: getFlagString(flags, "export-labeled"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
reason: parseReviewReason(getFlagString(flags, "reason")),
|
||||
resetPending: hasFlag(flags, "reset-pending"),
|
||||
});
|
||||
console.info(
|
||||
`[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`,
|
||||
);
|
||||
} finally {
|
||||
await garnet.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "calibrate") {
|
||||
const { getEvalCalibrationMinCorr } = await import("../config.js");
|
||||
const { computeCalibrationReport } = await import(
|
||||
"./review/calibration.js"
|
||||
);
|
||||
const mongo = await connectMongoForEval();
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
const report = await computeCalibrationReport(
|
||||
mongo.connection,
|
||||
garnet.garnetClient,
|
||||
{
|
||||
sinceMs: getFlagString(flags, "since")
|
||||
? parseDurationMs(getFlagString(flags, "since"))
|
||||
: undefined,
|
||||
},
|
||||
);
|
||||
printCalibrationReport(report);
|
||||
if (report.pairs < 10) {
|
||||
console.warn(
|
||||
"[eval] calibration pairs<10; check gate is informational until at least 10 labels are available",
|
||||
);
|
||||
}
|
||||
if (
|
||||
hasFlag(flags, "check") &&
|
||||
report.pairs >= 10 &&
|
||||
report.pearsonOverall < getEvalCalibrationMinCorr()
|
||||
) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} finally {
|
||||
await garnet.close();
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "propose") {
|
||||
const { createEvalLlmClient } = await import("./runner.js");
|
||||
const {
|
||||
buildProposalInputFromArchive,
|
||||
proposeVariants,
|
||||
writePromptVariantProposal,
|
||||
} = await import("./loop/propose.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const input = await buildProposalInputFromArchive(
|
||||
mongo.connection,
|
||||
parseSlice(getFlagString(flags, "slice")),
|
||||
{
|
||||
force: hasFlag(flags, "force"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
},
|
||||
);
|
||||
const proposals = await proposeVariants(createEvalLlmClient(), input);
|
||||
for (const proposal of proposals.prompts) {
|
||||
const outputPath = await writePromptVariantProposal(
|
||||
proposal,
|
||||
getFlagString(flags, "output-dir"),
|
||||
);
|
||||
console.info(
|
||||
`[eval] proposal variant=${proposal.variantId} output=${outputPath}`,
|
||||
);
|
||||
}
|
||||
for (const tool of proposals.tools) {
|
||||
console.info(
|
||||
`[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "tournament") {
|
||||
const { createEvalLlmClient } = await import("./runner.js");
|
||||
const {
|
||||
loadTournamentCases,
|
||||
runTournament,
|
||||
saveTournamentResult,
|
||||
} = await import("./loop/tournament.js");
|
||||
const variantIds = parseCsv(
|
||||
getFlagString(flags, "variants") ?? positionals[0],
|
||||
);
|
||||
if (variantIds.length < 2) {
|
||||
throw new Error(
|
||||
'eval:tournament requires --variants=control,<variantId>',
|
||||
);
|
||||
}
|
||||
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
||||
const cases = await loadTournamentCases(mongo.connection, {
|
||||
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
||||
limit: maxMatches,
|
||||
});
|
||||
const result = await runTournament(createEvalLlmClient(), {
|
||||
cases,
|
||||
maxMatches,
|
||||
variantIds,
|
||||
});
|
||||
await saveTournamentResult(mongo.connection, result);
|
||||
printTournamentResult(result);
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "tournament-results") {
|
||||
const { queryTournamentResults } = await import("./loop/tournament.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const results = await queryTournamentResults(mongo.connection, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
variantId: getFlagString(flags, "variant"),
|
||||
});
|
||||
for (const result of results) {
|
||||
printTournamentResult(result);
|
||||
}
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown eval command: ${command}`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { EvalArchiveEntry } from "./archive.js";
|
||||
import { computeDriftReportsFromEntries } from "./drift.js";
|
||||
|
||||
function archiveEntry(
|
||||
evaluatedAt: Date,
|
||||
overall: number,
|
||||
): EvalArchiveEntry {
|
||||
return {
|
||||
streamId: `${evaluatedAt.toISOString()}:${overall}`,
|
||||
evaluatedAt,
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
},
|
||||
response: {
|
||||
requestId: "req-1",
|
||||
text: "응답",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "응답",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: overall,
|
||||
accuracy: overall,
|
||||
concision: overall,
|
||||
scope: overall,
|
||||
overall,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("computeDriftReportsFromEntries", () => {
|
||||
it("marks a slice significant when the recent window drops enough", () => {
|
||||
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
|
||||
const recent = Array.from({ length: 30 }, (_, index) =>
|
||||
archiveEntry(new Date(nowMs - index * 60_000), 2),
|
||||
);
|
||||
const older = Array.from({ length: 40 }, (_, index) =>
|
||||
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000 - index * 60_000), 4),
|
||||
);
|
||||
|
||||
const reports = computeDriftReportsFromEntries([...recent, ...older], {
|
||||
nowMs,
|
||||
thresholdPct: 15,
|
||||
});
|
||||
|
||||
expect(reports).toHaveLength(1);
|
||||
expect(reports[0]).toMatchObject({
|
||||
recent24hAverage: 2,
|
||||
recent24hCount: 30,
|
||||
recent7dCount: 70,
|
||||
significant: true,
|
||||
});
|
||||
expect(reports[0]?.dropPct).toBeGreaterThan(15);
|
||||
});
|
||||
|
||||
it("requires enough recent samples for significance", () => {
|
||||
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
|
||||
const reports = computeDriftReportsFromEntries(
|
||||
[
|
||||
archiveEntry(new Date(nowMs), 1),
|
||||
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000), 5),
|
||||
],
|
||||
{
|
||||
nowMs,
|
||||
thresholdPct: 15,
|
||||
},
|
||||
);
|
||||
|
||||
expect(reports[0]?.significant).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
import { getAlertThresholdDriftPct } from "../config.js";
|
||||
import type {
|
||||
EvalArchiveEntry,
|
||||
MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { getEvalArchiveCollection } from "./archive.js";
|
||||
import type { SliceKey } from "./aggregator.js";
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const RECENT_WINDOW_MS = DAY_MS;
|
||||
const BASELINE_WINDOW_MS = 7 * DAY_MS;
|
||||
const MIN_SIGNIFICANT_RECENT_COUNT = 30;
|
||||
|
||||
export interface DriftReport {
|
||||
slice: SliceKey;
|
||||
recent24hAverage: number;
|
||||
recent7dAverage: number;
|
||||
dropPct: number;
|
||||
significant: boolean;
|
||||
recent24hCount: number;
|
||||
recent7dCount: number;
|
||||
}
|
||||
|
||||
interface DriftGroup {
|
||||
baselineValues: number[];
|
||||
recentValues: number[];
|
||||
slice: SliceKey;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function average(values: number[]): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function sliceKeyToString(slice: SliceKey): string {
|
||||
return `${slice.tier}:${slice.intent}:${slice.phase}`;
|
||||
}
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
return {
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
};
|
||||
}
|
||||
|
||||
function getMetricValue(
|
||||
entry: EvalArchiveEntry,
|
||||
metricKey: "llmOverall" | "validatorPassRate",
|
||||
): number | null {
|
||||
if (metricKey === "validatorPassRate") {
|
||||
return entry.validation.status === "pass" ? 1 : 0;
|
||||
}
|
||||
|
||||
if (entry.llmJudge.skipped) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
export function computeDriftReportsFromEntries(
|
||||
entries: EvalArchiveEntry[],
|
||||
options?: {
|
||||
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
|
||||
nowMs?: number | undefined;
|
||||
thresholdPct?: number | undefined;
|
||||
},
|
||||
): DriftReport[] {
|
||||
const metricKey = options?.metricKey ?? "llmOverall";
|
||||
const nowMs = options?.nowMs ?? Date.now();
|
||||
const recentCutoff = nowMs - RECENT_WINDOW_MS;
|
||||
const thresholdPct = options?.thresholdPct ?? getAlertThresholdDriftPct();
|
||||
const groups = new Map<string, DriftGroup>();
|
||||
|
||||
for (const entry of entries) {
|
||||
const value = getMetricValue(entry, metricKey);
|
||||
if (value === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const evaluatedAtMs = entry.evaluatedAt.getTime();
|
||||
if (!Number.isFinite(evaluatedAtMs)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const slice = getSlice(entry);
|
||||
const key = sliceKeyToString(slice);
|
||||
const group =
|
||||
groups.get(key) ??
|
||||
{
|
||||
baselineValues: [],
|
||||
recentValues: [],
|
||||
slice,
|
||||
};
|
||||
|
||||
group.baselineValues.push(value);
|
||||
if (evaluatedAtMs >= recentCutoff) {
|
||||
group.recentValues.push(value);
|
||||
}
|
||||
groups.set(key, group);
|
||||
}
|
||||
|
||||
return [...groups.values()].flatMap((group) => {
|
||||
const recentAverage = average(group.recentValues);
|
||||
const baselineAverage = average(group.baselineValues);
|
||||
if (recentAverage === null || baselineAverage === null || baselineAverage <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const dropPct = ((baselineAverage - recentAverage) / baselineAverage) * 100;
|
||||
|
||||
return [
|
||||
{
|
||||
slice: group.slice,
|
||||
recent24hAverage: round(recentAverage),
|
||||
recent7dAverage: round(baselineAverage),
|
||||
dropPct: round(dropPct),
|
||||
significant:
|
||||
group.recentValues.length >= MIN_SIGNIFICANT_RECENT_COUNT &&
|
||||
dropPct >= thresholdPct,
|
||||
recent24hCount: group.recentValues.length,
|
||||
recent7dCount: group.baselineValues.length,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export async function computeDriftReports(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
|
||||
nowMs?: number | undefined;
|
||||
},
|
||||
): Promise<DriftReport[]> {
|
||||
const nowMs = options?.nowMs ?? Date.now();
|
||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
evaluatedAt: {
|
||||
$gte: new Date(nowMs - BASELINE_WINDOW_MS),
|
||||
},
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(50_000)
|
||||
.toArray();
|
||||
|
||||
return computeDriftReportsFromEntries(entries, {
|
||||
metricKey: options?.metricKey,
|
||||
nowMs,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
|
||||
{"id":"case_119e66ab","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777693377499-0; llmOverall=1.5; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=카카오톡 닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 면정학/89/남/D처럼 쓰시면 돼요.","input":{"request":{"requestId":"req_7e7337de","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"닉네임 변경 및 하트 인증은 하였는데, 혹시 어느 부분에서 문제가 있는지 여쭈어 볼 수 있을까요?"}},"expected":{}}
|
||||
{"id":"case_6bc9858f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777692194987-0; llmOverall=1.5; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=공지방은 따로 운영하지 않습니다.","input":{"request":{"requestId":"req_c91897cf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"죄송합니다. 제가 기억하기론 공지방이 따로 있던 것으로 기억하는데, 공지방은 따로 없는지 여쭈어 볼 수 있을까요?"}},"expected":{}}
|
||||
{"id":"case_edbec2e7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777678836132-0; llmOverall=2; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=디스코드의 주민신청 채널에서 멘션 남기면 됩니다.","input":{"request":{"requestId":"req_e69bfe09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_2096a462","nickname":"<nick:e63f2878>","trigger":"user-message","userMessage":"디코 공지 확인이랑 닉변까지 다 했는데 어디에서 멘션하면 되나여"}},"expected":{}}
|
||||
{"id":"case_756da19b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693260088-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=아, 그러셨군요. 지금은 카카오톡 닉네임 변경만 남아 있어서 그 과정에서 막히신 거죠. \n무슨 부분이 어려우신가요? 어떤 과정에서 막히셨는지 알려 주세요.","input":{"request":{"requestId":"req_f09cd933","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"하는데"}},"expected":{}}
|
||||
{"id":"case_70b3db02","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693242995-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=네, 배치 보려면 먼저 카카오톡 1단계부터 끝내야 해요. 현재는 닉네임 변경만 남아 있고, 마감이 임박했으니 지금 바로 닉네임만 먼저 완료해 주세요.","input":{"request":{"requestId":"req_b68c3a89","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"얼른 배치를 봐야만"}},"expected":{}}
|
||||
{"id":"case_d8efa98f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777692242389-0; llmOverall=3.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 맞습니다. 구인글은 구인방에서만 올리는 게 원칙이에요.","input":{"request":{"requestId":"req_027cab09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"감사합니다.\n구인방에서만 구인글을 게시하는지 마지막으로 여쭈어 볼 수 있을까요?"}},"expected":{}}
|
||||
{"id":"case_b93a3eac","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777691863103-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=하트 인증은 이렇게만 다시 보내면 돼요.\n\n채팅방 오른쪽 상단 메뉴 버튼 → ‘커버 보기’ 들어가기 \n하트를 빨간색으로 바꾸기 \n그 상태를 편집 없이 원본 그대로 캡처해서 사진으로 보내기\n\n자르기나 화면 꾸미기 같은 편집은 하지 말고, 그냥 바로 스크린샷 그대로 보내주세요.","input":{"request":{"requestId":"req_1785e212","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_4c4e71d6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655309842-0; llmOverall=4.75; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=무슨 말인지 모르겠어용 ㅎㅎ 한 번만 더 정확히 말해줄 수 있어임니당?","input":{"request":{"requestId":"req_1147b077","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"<nick:8a181c76>","trigger":"user-message","userMessage":"머징"}},"expected":{}}
|
||||
{"id":"case_b3ba68ec","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655250210-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=오키, 점수 200점 확인했어용! 잘 적어둘게요.","input":{"request":{"requestId":"req_1f66afdc","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"<nick:8a181c76>","trigger":"user-message","userMessage":"200점수따리라"}},"expected":{}}
|
||||
{"id":"case_475ee343","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777649519625-0; llmOverall=4.75; derivedIntent=acknowledgement; derivedPhase=completed; assistantResponse=네, 다 잘 끝났으니까 푹 쉬었다 다녀오세요~","input":{"request":{"requestId":"req_11add9b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57a7acdc","nickname":"<nick:484b87ad>","trigger":"user-message","userMessage":"가겠습니당~~"}},"expected":{}}
|
||||
{"id":"case_3029ecb9","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777778911429-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 카카오톡 구인방 입장, 디스코드 서버 닉네임 설정, 디스코드 주민신청 채널 메시지 남기기예요.","input":{"request":{"requestId":"req_d1ec54d0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"<nick:df121b40>","trigger":"user-message","userMessage":"ai인가요 저분"}},"expected":{}}
|
||||
{"id":"case_65a019a2","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777786270802-0; llmOverall=1.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=27세 이상이면 지금은 2가지만 하면 돼요. \n카카오톡 닉네임 형식으로 변경하고, 방 하트 인증을 완료해 주세요. \n카카오톡 닉네임 형식은 이름/출생 연도 두 자리/성별/티어이고, 방 하트 인증은 채팅방 오른쪽 상단 메뉴의 '커버 보기'로 들어가 하트를 빨간색으로 바꾼 뒤 편집 없는 원본 스크린샷을 올려주시면 돼요.","input":{"request":{"requestId":"req_d5b6165a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_911011bd","nickname":"<nick:a9aef913>","trigger":"user-message","userMessage":"가입하려구하는데어케하나요?"}},"expected":{}}
|
||||
{"id":"case_60504e1f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777711499377-0; llmOverall=1.5; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 디스코드 주민신청 채널에 메시지 남기기만 완료하시면 됩니다.","input":{"request":{"requestId":"req_a50a581c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ae8873b8","nickname":"<nick:5cf8dc88>","trigger":"user-message","userMessage":"안녕하세요 :)\n오늘 내전 있을까요?"}},"expected":{}}
|
||||
{"id":"case_a912b1f1","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777708255924-0; llmOverall=1.5; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=지금은 카카오톡 1단계(닉네임/방 하트 인증) 조건이 먼저 필요해요. \n현재 남은 항목은 방 하트 인증이고 아직 미완료 상태라, 이게 끝나야 디스코드 안내를 이어드릴 수 있어요.","input":{"request":{"requestId":"req_7ed0470a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c6e63b5e","nickname":"<nick:6fbed163>","trigger":"user-message","userMessage":"/디코"}},"expected":{}}
|
||||
{"id":"case_1742409e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777778244667-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임이 안 되는 이유는 티어가 빠졌기 때문이에요. \n형식은 닉네임/출생연도/성별/티어이고, 지금 망태/95/남 뒤에 /D 같은 한 글자 티어만 추가하면 됩니다.","input":{"request":{"requestId":"req_cb17dffc","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"<nick:2c439ed8>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_1ae6d8a5","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777768915244-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 형식이 맞지 않아서 안 됩니다. \n3렙탑갱필킬/ 뒤에 출생 연도 두 자리, 성별, 티어를 이름/연도/성별/티어 순서로 채워야 해요. \n예를 들어 3렙탑갱필킬/89/남/D 형태로 수정하면 됩니다.","input":{"request":{"requestId":"req_ec4696f6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0b248a72","nickname":"<nick:60e95b64>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_a47d12f7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777708249627-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 너뒤에내가있다/98/남은 형식에서 마지막 티어가 빠져서 안 되는 거예요. \n닉네임/출생연도/성별/티어로 맞추려면 예) 너뒤에내가있다/98/남/D처럼 티어를 하나 추가하면 됩니다. \n티어는 GM, I, B, S, G, P, E, M, C, U 중 하나로 넣으면 돼요.","input":{"request":{"requestId":"req_0497f34d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c6e63b5e","nickname":"<nick:66e67a5f>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_fe83f154","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777710710256-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_47316f87","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_aaa91459","nickname":"<nick:8e449d55>","trigger":"user-message","userMessage":"넵"}},"expected":{}}
|
||||
{"id":"case_07b6910d","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777778857540-0; llmOverall=2.75; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=구인방은 구인/구해요 글을 올리거나 확인하는 방이야.","input":{"request":{"requestId":"req_1480e5f2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"<nick:df121b40>","trigger":"user-message","userMessage":"/구인방"}},"expected":{}}
|
||||
{"id":"case_2e1c9c7d","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777711775572-0; llmOverall=2.75; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 잘 마무리하셨어요. 천천히 잘 마쳤네요.","input":{"request":{"requestId":"req_d2a86b68","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_dfd55a6d","nickname":"<nick:2e84bbf2>","trigger":"user-message","userMessage":"이제야 다 끝냈넹요,,"}},"expected":{}}
|
||||
{"id":"case_c3be2795","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777778838385-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=완료된 건 빼고 남은 건 3개예요. \n카카오톡 구인방 입장, 디스코드 서버 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기.","input":{"request":{"requestId":"req_af508d85","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"<nick:df121b40>","trigger":"user-message","userMessage":"아이게아닌가"}},"expected":{}}
|
||||
{"id":"case_14386176","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777778305118-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=방하트 인증은 새내기 등록 1단계에서 채팅방 오른쪽 상단의 하트를 빨간색으로 바꿔 두고, 그 화면이 보이게 편집 없이 원본 스크린샷으로 보내는 인증이에요. \n방 하트 인증은 등록 1단계의 일부에 해당해요.","input":{"request":{"requestId":"req_6a17948f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"<nick:d4c7debd>","trigger":"user-message","userMessage":"방하트인증은뭔가요?"}},"expected":{}}
|
||||
{"id":"case_56116efc","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777778262296-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요. 편하게 진행해 주세요.","input":{"request":{"requestId":"req_f8ff4569","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f213fc7","nickname":"<nick:2c439ed8>","trigger":"user-message","userMessage":"넹"}},"expected":{}}
|
||||
{"id":"case_d44b5677","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777769958583-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 카카오톡 닉네임 변경과 방 하트 인증입니다.","input":{"request":{"requestId":"req_ad0a173e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_78fa631c","nickname":"<nick:cd84789b>","trigger":"user-message","userMessage":"아이언인데 괜찮나요"}},"expected":{}}
|
||||
{"id":"case_42167e0d","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777769920617-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=어서오세요!","input":{"request":{"requestId":"req_525ed39b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_78fa631c","nickname":"<nick:cd84789b>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_09834f65","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777768954579-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요, 반갑습니다!","input":{"request":{"requestId":"req_5763eb31","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_0b248a72","nickname":"<nick:71807197>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_f5695075","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777863604872-0; llmOverall=1.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=안 되는 이유는 성별(남/여) 칸이 빠져서예요. \n낑꽁껑꿍#2509/99/D에서 99와 D 사이에 남 또는 여를 넣어 \n낑꽁껑꿍#2509/99/남/D처럼 고치면 됩니다.","input":{"request":{"requestId":"req_49fcd14f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40adc660","nickname":"<nick:20d901d7>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_aaaaf8dc","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777876463824-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 “긁적이는 춘식이”는 이름만 있어요. \n규칙은 '/'로 이름/출생연도 두 자리/성별/티어 4칸이 들어가야 하는데, 지금은 1칸뿐이라 형식 오류예요. \n\n이름 뒤에 출생연도 두 자리/성별/티어를 붙여서 \n예) 춘식이/95/남/D 처럼 바꿔주면 돼요.","input":{"request":{"requestId":"req_fdb2a403","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c61645af","nickname":"<nick:d6894c42>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_64c23ac4","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777876455103-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임 이동윤이 안 되는 이유는 형식이 안 맞아서예요. \n지금은 이름/출생연도 두 자리/성별/티어 중에서 이름만 있고 연도·성별·티어가 빠짐. \n\n이동윤/89/남/D처럼 이름/출생연도 두 자리/성별/티어 순서로 바꿔주면 돼요.","input":{"request":{"requestId":"req_b104b41f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c61645af","nickname":"<nick:31600c8c>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_ae2f0863","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777876377377-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=어서오세요!","input":{"request":{"requestId":"req_0f32d5e8","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_c61645af","nickname":"<nick:3142b96d>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_537bd6bd","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777863619306-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_1582e6b5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_40adc660","nickname":"<nick:20d901d7>","trigger":"user-message","userMessage":"넵 잠시만요!"}},"expected":{}}
|
||||
{"id":"case_e57f3c00","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777833126526-0; llmOverall=n/a; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=네, 지금은 디스코드 승인 진행이 가능합니다. \n최종 등록 완료는 디스코드 관련 항목(서버 프로필 닉네임 설정, 주민신청 채널 메시지 남기기)과 카카오톡 본인인증방/구인방 입장이 모두 끝났을 때 확정돼요.","input":{"request":{"requestId":"req_9887396c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"<nick:c3673db1>","trigger":"user-message","userMessage":"혹시 디스코드 승인 가능한가요?"}},"expected":{}}
|
||||
{"id":"case_d736890c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777831792635-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 편하게 진행해 주세요.","input":{"request":{"requestId":"req_97da2b0d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"<nick:298be8d3>","trigger":"user-message","userMessage":"네"}},"expected":{}}
|
||||
{"id":"case_86ed5b26","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777831774469-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=연화/99/남/D 형식은 규칙에 맞습니다. \n카카오톡 닉네임으로 사용하면 됩니다.","input":{"request":{"requestId":"req_c3783067","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"<nick:298be8d3>","trigger":"user-message","userMessage":"연화/99/남/D"}},"expected":{}}
|
||||
{"id":"case_af091d04","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777831748947-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=연화/99/남/D3는 티어가 규칙과 맞지 않습니다. \n티어는 GM, I, B, S, G, P, E, D, M, C, U 하나만 사용해 주세요. 예: 연화/99/남/D","input":{"request":{"requestId":"req_d833b83a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57d6cdbd","nickname":"<nick:298be8d3>","trigger":"user-message","userMessage":"연화/99/남/D3"}},"expected":{}}
|
||||
{"id":"case_e138ce19","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777958026869-0; llmOverall=1.25; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=지금은 카카오톡 1단계가 아직 끝나지 않아서(닉네임 형식 수정 미완료) 디스코드 안내는 진행할 수 없어요. \n닉네임을 먼저 바꿔서 1단계를 완료하면 디스코드 쪽 안내해드릴게요.","input":{"request":{"requestId":"req_92962005","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"<nick:fb83304e>","trigger":"user-message","userMessage":"디스코드"}},"expected":{}}
|
||||
{"id":"case_361b1834","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906129199-0; llmOverall=1.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임에서 안 되는 이유는 마지막 티어가 소문자 e라서예요. \n티어 코드는 대문자만 허용되니 .../여/E처럼 마지막 부분만 대문자 E로 바꿔주시면 됩니다.","input":{"request":{"requestId":"req_9b857f60","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"<nick:73447ea0>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_729b540d","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777958095517-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=좋아요, 재롱잉/99/남/M은 카카오톡 닉네임 형식에 맞아요. 카카오톡 1단계(닉네임 변경)는 완료됐어요.","input":{"request":{"requestId":"req_9e72e7c1","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"<nick:fb83304e>","trigger":"user-message","userMessage":"재롱잉/99/남/M"}},"expected":{}}
|
||||
{"id":"case_c205d59f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906018385-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_a52d560b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"<nick:06340619>","trigger":"user-message","userMessage":"롤닉적는건가요?"}},"expected":{}}
|
||||
{"id":"case_cb61358e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777886150774-0; llmOverall=1.5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요! 요청하신 형식대로 JSON 객체만 다시 출력했어요","input":{"request":{"requestId":"req_23d248e5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_9a2e13c5","nickname":"<nick:f83e502c>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_7db8c4dd","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777886215575-0; llmOverall=1.75; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코는 지금 바로 진행하면 됩니다. 카카오톡 1단계(닉네임 변경/하트 인증)는 이미 끝났고, 이제 디스코드로 넘어가시면 돼요. \n현재는 본인인증방/구인방 입장과 디스코드 서버 프로필 설정, 그리고 주민신청 채널에 메시지 남기기가 남아 있어요.","input":{"request":{"requestId":"req_5e290120","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_9a2e13c5","nickname":"<nick:f53a2e41>","trigger":"user-message","userMessage":"디코는"}},"expected":{}}
|
||||
{"id":"case_da5eb130","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906410144-0; llmOverall=2; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=그건 닉네임 뒤에 붙는 티어 코드야. \n티어 코드는 GM, I, B, S, G, P, E, D, M, C, U 중 하나만 대문자로 쓰면 돼. \n지금은 뒤의 e 대신 E처럼 대문자로 바꿔주면 해결돼.","input":{"request":{"requestId":"req_4e5c8c85","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"<nick:5233cf2c>","trigger":"user-message","userMessage":"그게몬가여"}},"expected":{}}
|
||||
{"id":"case_27613760","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906039837-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=닉네임 길이 자체는 괜찮아요. 다만 형식은 꼭 '이름/출생 연도 두 자리/성별/티어' 4칸으로 맞춰야 해요. \n이름이 길면 줄여서 입력해도 됩니다.","input":{"request":{"requestId":"req_ade6fcdf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"<nick:06340619>","trigger":"user-message","userMessage":"닉네임이 엄청긴데.."}},"expected":{}}
|
||||
{"id":"case_9bf50f92","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777906011838-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=@1111님 닉네임을 조금만 수정해 주세요!\n'/'로 구분된 4개 항목(이름/나이/성별/티어)이 필요해요. 현재 1개입니다.","input":{"request":{"requestId":"req_b87c7d1a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3cef8b72","nickname":"<nick:06340619>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_3be52429","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777887243301-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코 신청해주신 것 확인했어요. \n지금은 남은 항목이 있어서 아직 전체 등록이 마무리되지 않았습니다. \n남은 항목: 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널 메시지 남기기","input":{"request":{"requestId":"req_b7c0c89d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_9a2e13c5","nickname":"<nick:f53a2e41>","trigger":"user-message","userMessage":"디코 신청 했습니다"}},"expected":{}}
|
||||
{"id":"case_ce59eef5","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777958037609-0; llmOverall=n/a; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 변경한 카카오톡 닉네임을 보내주시면 바로 확인할게요.","input":{"request":{"requestId":"req_e32dd182","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"<nick:fb83304e>","trigger":"user-message","userMessage":"넣어놨는데"}},"expected":{}}
|
||||
{"id":"case_c0d8c07b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777957698916-0; llmOverall=3.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=재롱잉#0606은 카카오톡 닉네임 형식이 아니에요. \n#는 형식에 맞지 않고, 연도/성별/티어 항목이 비어 있어요. \n카카오톡 닉네임의 '재롱잉' 뒤에 연도(2자리)/성별/티어를 순서대로 써서 고치면 됩니다.","input":{"request":{"requestId":"req_9a0ce4a3","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_56ec7104","nickname":"<nick:fb83304e>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_9a324a7b","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980725744-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=이제 카카오톡 본인인증방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 남아 있어요.","input":{"request":{"requestId":"req_7cb8c7b8","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"<nick:7c96607d>","trigger":"user-message","userMessage":"방금 들어가긴 했는데 된건가여"}},"expected":{}}
|
||||
{"id":"case_ef5a5a39","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980147468-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=현재 남은 건 카카오톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 그리고 디스코드 주민신청 채널에 메시지 남기기야.","input":{"request":{"requestId":"req_15b6a97b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"<nick:7c96607d>","trigger":"user-message","userMessage":"정독 후 변경 완료 했는데 혹시 제가 놓친 부분이 있을까여"}},"expected":{}}
|
||||
{"id":"case_49c7d986","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777976015637-0; llmOverall=1; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 카카오톡 본인인증방 입장, 카카오톡 구인방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기예요.","input":{"request":{"requestId":"req_b22f9f8a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_35175df8","nickname":"<nick:1785179a>","trigger":"user-message","userMessage":"안녕하세용 자랭 한 분 가능하신 사람 있으신가요?? 서폿이 비어서요!!"}},"expected":{}}
|
||||
{"id":"case_0313331d","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777997633396-0; llmOverall=1.5; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=좋아요, 하트 인증 완료로 확인했어요.","input":{"request":{"requestId":"req_ea94198d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_8dfc83e0","nickname":"<nick:ed84ea14>","trigger":"user-message","userMessage":"하트 했습니다!"}},"expected":{}}
|
||||
{"id":"case_057e6219","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777976055251-0; llmOverall=1.5; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디스코드를 안 하면 새내기 등록이 아직 완료되지 않습니다. \n지금 남은 건 카카오톡 본인인증방/구인방 입장하고, 디스코드에서 서버 닉네임 설정과 주민신청 채널 메시지 남기기 두 개라서 필수로 처리해야 해요.","input":{"request":{"requestId":"req_201d5367","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_35175df8","nickname":"<nick:1785179a>","trigger":"user-message","userMessage":"제가 디스코드를 안하는데,,"}},"expected":{}}
|
||||
{"id":"case_e5b73b18","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778043112369-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=잘하셨어요. 디코 신청까지 진행한 거 확인했어요. 지금 남은 건 카카오톡 본인인증방 입장, 디스코드 서버 닉네임 설정, 주민신청 채널 메시지 남기기예요. 마감 전에 3가지만 빨리 마무리하면 돼요.","input":{"request":{"requestId":"req_ba7b68e0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ef6b0188","nickname":"<nick:c5581352>","trigger":"user-message","userMessage":"디코 신청까지 했습미당🙏🏻"}},"expected":{}}
|
||||
{"id":"case_f2bb8318","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777993150807-0; llmOverall=2; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 카카오톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기입니다.","input":{"request":{"requestId":"req_7ee35c98","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_6f2bef29","nickname":"<nick:9cb53098>","trigger":"user-message","userMessage":"이렇게 하면 되나용"}},"expected":{}}
|
||||
{"id":"case_8bf55b0b","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980262565-0; llmOverall=2; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요ㅎㅎ 채팅방 상단 고정 공지에 있는 입장 버튼 누르면 바로 들어가져요.","input":{"request":{"requestId":"req_8ca31700","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"<nick:7c96607d>","trigger":"user-message","userMessage":"예?.."}},"expected":{}}
|
||||
{"id":"case_3b3eeed7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777980240472-0; llmOverall=2; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=채팅방 상단 고정 공지에 있는 카카오톡 본인인증방 입장 버튼 누르면 바로 들어갈 수 있어.","input":{"request":{"requestId":"req_957b3c03","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_513ed6f3","nickname":"<nick:7c96607d>","trigger":"user-message","userMessage":"카카오톡 본인인증은 어디서 하나여? 처음이라ㅠㅠ"}},"expected":{}}
|
||||
{"id":"case_11779215","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777975920326-0; llmOverall=2.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임이 성별/티어 형식을 안 맞춰서 안 됩니다. \n어피치/00/M에서 M은 남 또는 여로 바꿔야 하고, 끝에 티어 한 글자(GM, I, B, S, G, P, E, D, M, C, U)를 추가해야 해요. \n예시: 어피치/00/남/D","input":{"request":{"requestId":"req_f3956f12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_35175df8","nickname":"<nick:81b8b98b>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_5e681f63","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777998375356-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 마저 진행해 주세요.","input":{"request":{"requestId":"req_2adf33e7","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_8dfc83e0","nickname":"<nick:ed84ea14>","trigger":"user-message","userMessage":"넵 알겠습니다"}},"expected":{}}
|
||||
{"id":"case_e714d976","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777997006108-0; llmOverall=2.75; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임은 형식이 한 칸 부족해요. \nAsolOX/97/M은 이름/출생연도/성별/티어 순서에서 성별 뒤에 티어가 빠져 있고, M도 성별 표기로는 안 됩니다.\n\n성별을 남 또는 여로 쓰고, 마지막에 티어 한 글자(예: D)를 붙여서 AsolOX/97/남/D처럼 바꾸면 돼요.","input":{"request":{"requestId":"req_9684219b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_8dfc83e0","nickname":"<nick:c89a8059>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_6e1c52ff","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1777998239663-0; llmOverall=3; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=네, 다시 시도할 때는 아래만 지켜주세요. \n1) 채팅방 오른쪽 상단 메뉴 → 커버 보기 \n2) 하트를 다시 빨간색으로 변경 \n3) 바로 그 화면을 편집 없이 원본으로 새로 캡처해서 다시 업로드\n\n이렇게 하면 다시 인증 요청으로 처리돼요.","input":{"request":{"requestId":"req_7b09ac12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"<nick:9754d18b>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_745fddda","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778043731233-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 입장 잘 마무리하셨어요. 편하게 진행해 주세요.","input":{"request":{"requestId":"req_28d12b61","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ef6b0188","nickname":"<nick:c5581352>","trigger":"user-message","userMessage":"입장절차 끝..!! 잘부탁드려용!!"}},"expected":{}}
|
||||
{"id":"case_6ad937b9","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778018572746-0; llmOverall=5; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=네, 여기다 올리면 됩니다.","input":{"request":{"requestId":"req_847b39a1","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_b874c5cb","nickname":"<nick:e82d18ad>","trigger":"user-message","userMessage":"안녕하세요 여기다 올리면 될까요?"}},"expected":{}}
|
||||
{"id":"case_3d833f9f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998586336-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=건강/99/남/U님 카카오톡 절차는 완료됐고, 이제 카카오톡 본인인증방 입장, 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기만 진행하시면 돼요!","input":{"request":{"requestId":"req_ea34474f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"<nick:9754d18b>","trigger":"user-message","userMessage":"첨보네요"}},"expected":{}}
|
||||
{"id":"case_dffda8fa","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998570053-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.","input":{"request":{"requestId":"req_7268a270","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"<nick:9754d18b>","trigger":"user-message","userMessage":"프사도 있는건"}},"expected":{}}
|
||||
{"id":"case_ee4deca6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998289607-0; llmOverall=5; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=네, 닉네임 인증은 완료됐어요.","input":{"request":{"requestId":"req_42e6f27a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"<nick:9754d18b>","trigger":"user-message","userMessage":"저닉네임인증된거맞나요"}},"expected":{}}
|
||||
{"id":"case_c7a3bcc7","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998231025-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 올릴 수 있어요. 하트 인증은 편집 없이 원본 스크린샷으로 업로드하면 돼요.","input":{"request":{"requestId":"req_de2331dd","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"<nick:9754d18b>","trigger":"user-message","userMessage":"사진올림되나요"}},"expected":{}}
|
||||
{"id":"case_bf5f3496","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777998213299-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse={\"message\":\"방 하트 인증은 채팅방 오른쪽 상단 메뉴 버튼에서 '커버 보기'로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.\"}","input":{"request":{"requestId":"req_b1312c5a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_ea38805a","nickname":"<nick:9754d18b>","trigger":"user-message","userMessage":"하트인증은"}},"expected":{}}
|
||||
{"id":"case_96b3e285","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778059340376-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_7b92ce89","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_38e0f76d","nickname":"<nick:8e21f494>","trigger":"user-message","userMessage":"감사합니다"}},"expected":{}}
|
||||
{"id":"case_a2ec1268","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778118929517-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디스코드 서버프로필 닉네임은 디스코드 안에서 보여지는 이름 형식이에요. 형식은 닉네임#태그/출생연도 두 자리/티어입니다. 예: 할리#프렌치불독/88/D. 카카오톡 형식(성별 포함)과 다릅니다.","input":{"request":{"requestId":"req_fffe630c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"<nick:7cfd256f>","trigger":"user-message","userMessage":"디코 서버프로필 닉네임이 뭘까요,,,,,,,,"}},"expected":{}}
|
||||
{"id":"case_1bfe6ee3","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778056450407-0; llmOverall=2; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 방 하트 인증만 남았습니다.","input":{"request":{"requestId":"req_9d77d436","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_e89d3a0d","nickname":"<nick:73d3efe2>","trigger":"user-message","userMessage":"여기서 듀오 구해두 될까요"}},"expected":{}}
|
||||
{"id":"case_c16428d0","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778119100551-0; llmOverall=2.25; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요. 계속 천천히 마무리해 보세요.","input":{"request":{"requestId":"req_9ffcfc63","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"<nick:7cfd256f>","trigger":"user-message","userMessage":"같아요"}},"expected":{}}
|
||||
{"id":"case_b2997e82","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119474172-0; llmOverall=2.75; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 2개예요. 디스코드 서버 프로필 닉네임 설정하고, 주민신청 채널에 메시지 남기기만 하면 돼요.","input":{"request":{"requestId":"req_51c17a6d","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"<nick:7cfd256f>","trigger":"user-message","userMessage":"방 티어가 대체적으로"}},"expected":{}}
|
||||
{"id":"case_49fe7b98","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119334116-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=네, 아직 남은 건 \n1) 디스코드 서버 프로필 닉네임 설정 \n2) 주민신청 채널에 메시지 남기기 \n둘 다예요.","input":{"request":{"requestId":"req_d068d781","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"<nick:7cfd256f>","trigger":"user-message","userMessage":"절차 완료인가요!?"}},"expected":{}}
|
||||
{"id":"case_d598f1d3","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119088788-0; llmOverall=n/a; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 잘 하셨어요.","input":{"request":{"requestId":"req_0b53acfa","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"<nick:7cfd256f>","trigger":"user-message","userMessage":"성공한것"}},"expected":{}}
|
||||
{"id":"case_f47d6cfb","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778119000121-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=아니요. 계정정보 별명이 아니라 디스코드 서버에서 보이는 서버 프로필 닉네임을 바꿔야 해요.","input":{"request":{"requestId":"req_58ba3c26","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc1814c4","nickname":"<nick:7cfd256f>","trigger":"user-message","userMessage":"계정정보에서 별명을 바꾸면 되는걸까요?"}},"expected":{}}
|
||||
{"id":"case_bf6cf546","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778086326681-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=현재 남은 절차는 카카오톡 닉네임 변경과 방 하트 인증 두 가지만 완료하시면 됩니다.","input":{"request":{"requestId":"req_0c7527af","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f728e162","nickname":"<nick:9d653159>","trigger":"user-message","userMessage":"카카오 채널 1초로그인 -> 마이쿠폰함 확인 -> 네이버페이 쿠폰 등록 -> 코드 입력\n\nhttps://place.cashnote.kr/login/referrals/h776U3JtDjT\n\n지금 받을 수 있는 쿠폰이 빠르게 소진되고 있어요. (*남은 수량 : 560장)"}},"expected":{}}
|
||||
{"id":"case_6101b64c","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778086319157-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=다시 찍을 때는 이렇게만 해주면 돼요.\n\n1) 채팅방 오른쪽 위 메뉴 버튼에서 \n2) “커버 보기” 들어가기 \n3) 방 하트를 빨간색으로 바꾸기 \n4) 방금 그 화면이 딱 보이게 원본 캡처본만 올리기 \n(편집, 크롭, 필터, 글자 추가 없이 그대로)","input":{"request":{"requestId":"req_4ad09644","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f728e162","nickname":"<nick:9d653159>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_33c66dba","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778075276814-0; llmOverall=n/a; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=지금 남은 건 딱 세 가지예요. 카카오톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기입니다.","input":{"request":{"requestId":"req_5494a948","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f08f2d3d","nickname":"<nick:4a835608>","trigger":"user-message","userMessage":"주민신청했는데 기다리면 되는 건가요"}},"expected":{}}
|
||||
{"id":"case_377bc44a","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778074538639-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse={\"message\":\"안녕하세요~\"}","input":{"request":{"requestId":"req_3f66d1f6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_f08f2d3d","nickname":"<nick:d6894c42>","trigger":"user-message","userMessage":"안녕하세여"}},"expected":{}}
|
||||
{"id":"case_cff93d4f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778202454309-0; llmOverall=1.75; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코까지 완료된 건 확인했어요. \n아직 카카오톡 닉네임 변경만 남아 있어서 1단계가 끝나지 않았어요. \n닉네임은 '공주야너랑살래/99/남/D'처럼 슬래시 형식으로 수정해 주세요.","input":{"request":{"requestId":"req_35185cf6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7bed5e81","nickname":"<nick:57dd89c8>","trigger":"user-message","userMessage":"디코까지 완료했습니다~"}},"expected":{}}
|
||||
{"id":"case_030c1b63","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778202545113-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=형식이 조금만 달라도 반영이 안 될 수 있어요. \n카톡 닉네임은 이름/출생연도 두 자리/성별/티어, 티어는 한 글자만 가능해서 형식 오차 시 미반영돼요.","input":{"request":{"requestId":"req_e288dc78","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_7bed5e81","nickname":"<nick:57dd89c8>","trigger":"user-message","userMessage":"닉변도 양식대로 변경했는데 아직 안바뀐거로 나오나요..?"}},"expected":{}}
|
||||
{"id":"case_583e4bf2","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778154690883-0; llmOverall=2; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 건 3개예요. \n카톡 본인인증방 입장, 디스코드 서버 프로필 닉네임 설정, 디스코드 주민신청 채널에 메시지 남기기입니다.","input":{"request":{"requestId":"req_e2a2a3e2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_13eeeed8","nickname":"<nick:3afa05fc>","trigger":"user-message","userMessage":"됐늘까요"}},"expected":{}}
|
||||
{"id":"case_7fd8f41e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778154403955-0; llmOverall=2.75; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임이 안 되는 이유는 형식이 맞지 않아서예요. \n현재는 \"/\"가 없어 4개 항목이 한 번도 분리되지 않았거든요.\n\n고칠 부분은 닉네임 형식만 맞추면 됩니다:\n이름 / 출생 연도 두 자리 / 성별 / 티어 \n(성별: 남 또는 여 / 티어: GM, I, B, S, G, P, E, D, M, C, U 중 1개)","input":{"request":{"requestId":"req_9ef80785","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_13eeeed8","nickname":"<nick:0d4ec069>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_e29b7e22","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778301741564-0; llmOverall=1.25; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=문제는 티어 부분이 소문자 p라서 안 되는 거예요. 티어는 지정된 코드의 대문자여야 해서 p를 P로 고치면 됩니다.\n티어 | p를 P로 | akaps/92/남/P","input":{"request":{"requestId":"req_5370e799","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3150904d","nickname":"<nick:cf02046e>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_8c1af121","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778260759554-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 그렇게 해주시면 돼요.","input":{"request":{"requestId":"req_0e99bbb6","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"<nick:707539a8>","trigger":"user-message","userMessage":"아아 넵 감사합니다"}},"expected":{}}
|
||||
{"id":"case_a9637997","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778260901486-0; llmOverall=2; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=아니요, 별도 양식은 따로 없어요. 방 하트 인증은 하트 화면 원본 캡처만 올리면 됩니다.","input":{"request":{"requestId":"req_57e5ef12","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"<nick:707539a8>","trigger":"user-message","userMessage":"양식이 따로 있나요?"}},"expected":{}}
|
||||
{"id":"case_b6f5a13e","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778240845275-0; llmOverall=2; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=형식은 닉네임/출생 연도 두 자리/성별/티어(한 글자)예요.\n현재 닉네임은 두 번째 칸이 연도 자리에 '별동대'이고, 성별과 티어 위치가 맞지 않아요.\n두 번째 칸을 2자리 출생연도로 바꾸고 마지막에 티어를 한 글자 추가하면 됩니다.","input":{"request":{"requestId":"req_076e8a92","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_de12d27e","nickname":"<nick:c450035a>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_87b58a6a","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778236553292-0; llmOverall=2; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 맞아요. \n방 하트 인증은 채팅방 오른쪽 상단의 \"커버 보기\"에서 하트를 빨간색으로 바꾼 뒤, 편집 없는 원본 화면을 스크린샷으로 보내면 돼요.","input":{"request":{"requestId":"req_0a071eba","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_27efff54","nickname":"<nick:1d0accc1>","trigger":"user-message","userMessage":"인증 이러케하면 되나용?"}},"expected":{}}
|
||||
{"id":"case_a55d7708","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778260808728-0; llmOverall=2.25; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 천천히 진행해 주세요.","input":{"request":{"requestId":"req_9aa4d55f","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"<nick:707539a8>","trigger":"user-message","userMessage":"감사합니다 몰랐어요"}},"expected":{}}
|
||||
{"id":"case_f3191b8e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778260885661-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_b99c1bfd","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"<nick:707539a8>","trigger":"user-message","userMessage":"아아 넵 알겠습니다"}},"expected":{}}
|
||||
{"id":"case_07b7087f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778309150052-0; llmOverall=2.5; derivedIntent=status-question; derivedPhase=deadline-near; assistantResponse=남은 절차는 방 하트 인증뿐입니다.","input":{"request":{"requestId":"req_81fc82d0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bda05adb","nickname":"<nick:5b5d30e3>","trigger":"user-message","userMessage":"될까요?"}},"expected":{}}
|
||||
{"id":"case_20b46c53","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778309136760-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=deadline-near; assistantResponse=네, 맞아요. \n채팅방 오른쪽 상단 메뉴에서 커버 보기 들어가서 하트를 빨간색으로 바꾼 뒤, 그 화면을 편집 없이 원본으로 스크린샷 찍어 업로드해 주세요.","input":{"request":{"requestId":"req_4933bc72","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_bda05adb","nickname":"<nick:5b5d30e3>","trigger":"user-message","userMessage":"방 하트 인증은 사진 보내면"}},"expected":{}}
|
||||
{"id":"case_d334d6dd","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778260937318-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_ca9a5bd2","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_22e4eefa","nickname":"<nick:707539a8>","trigger":"user-message","userMessage":"넵"}},"expected":{}}
|
||||
{"id":"case_a4345a1f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778255244619-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=nickname-rejected; assistantResponse=현재 닉네임에서 안 되는 부분은 성별과 티어 사이 구분자 /가 빠진 점이에요. \n스카피/98/남D → 스카피/98/남/D로 바꿔주시면 돼요.","input":{"request":{"requestId":"req_2507bb99","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_d8b4617f","nickname":"<nick:8873d583>","trigger":"nickname-invalid"}},"expected":{}}
|
||||
{"id":"case_401941a0","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778350051338-0; llmOverall=1.5; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=네, 지금이라도 마감 전이면 가능합니다. 마감 전까지만 가능하니 지금 바로 진행해 주세요.","input":{"request":{"requestId":"req_f59d7357","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"<nick:15d16dd8>","trigger":"user-message","userMessage":"률중인데,,"}},"expected":{}}
|
||||
{"id":"case_251723ce","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778322156175-0; llmOverall=1.5; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=좋아요, 지금 남은 건 카카오톡 본인인증방 입장뿐이에요.","input":{"request":{"requestId":"req_073f9f3b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"<nick:3b47a030>","trigger":"user-message","userMessage":"주민신청,단통방하트인증,구인방입장 완료햇습니다"}},"expected":{}}
|
||||
{"id":"case_0382016f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778320965167-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 천천히 진행해 주세요.","input":{"request":{"requestId":"req_28d990e5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"<nick:3b47a030>","trigger":"user-message","userMessage":"감사합니다"}},"expected":{}}
|
||||
{"id":"case_52c3e5d8","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778316434893-0; llmOverall=1.75; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=조아재/92/남/E님 이제 추가 절차만 남았어요. 디스코드 프로필 닉네임 설정과 디스코드 주민신청 채널에 메시지 남기기을 확인해 주세요.","input":{"request":{"requestId":"req_b0351ff3","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_04deef9f","nickname":"<nick:aa173f7f>","trigger":"user-message","userMessage":"디스코드는 조금 기다려야겠죠?"}},"expected":{}}
|
||||
{"id":"case_a03ff5a9","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778322017346-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=디코 승인은 디스코드 주민신청 채널에 메시지를 남겨야 신청이 접수돼요. \n현재는 카카오톡 1단계(닉네임/하트)가 끝났으니 여기서 바로 진행하시면 됩니다.","input":{"request":{"requestId":"req_b76d0382","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"<nick:3b47a030>","trigger":"user-message","userMessage":"관리자님~ 디코 승인부탁드립니다"}},"expected":{}}
|
||||
{"id":"case_33f24d6f","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778335070993-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요! 천천히 진행해 주세요.","input":{"request":{"requestId":"req_6948769b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_961ebb8c","nickname":"<nick:cfdcccbd>","trigger":"user-message","userMessage":"넵, 감사합니다."}},"expected":{}}
|
||||
{"id":"case_4f8acb72","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778350065869-0; llmOverall=3; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 확인했어요.","input":{"request":{"requestId":"req_2adcf9cb","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"<nick:15d16dd8>","trigger":"user-message","userMessage":"네ㅔㅔ"}},"expected":{}}
|
||||
{"id":"case_84729b2b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778392723529-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_77df97b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_5822e361","nickname":"<nick:a9e1543b>","trigger":"user-message","userMessage":"안녕하세요~"}},"expected":{}}
|
||||
{"id":"case_b27986e1","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778350033557-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=죄송하지만 지금은 마감이 임박해 있어요. 조금이라도 빨리 처리해 주세요. 마감 전까지만 가능합니다.","input":{"request":{"requestId":"req_626ca48c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"<nick:15d16dd8>","trigger":"user-message","userMessage":"죄송한데 조금만 이ㅛ다가 해도더ㅣ냐요?"}},"expected":{}}
|
||||
{"id":"case_8270994b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778348438021-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=먼저 닉네임 변경과 하트 인증 두 가지를 완료하시면 돼요. 톡 게시글의 공지를 확인해 주세요!","input":{"request":{"requestId":"req_518fd7a5","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"<nick:5b247222>","trigger":"user-message","userMessage":"네넨"}},"expected":{}}
|
||||
{"id":"case_32db0b90","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778348422100-0; llmOverall=n/a; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=아니요, 롤 닉네임이 아니라 카카오톡 닉네임을 이름/출생 연도 두 자리/성별/티어 형식으로 등록해야 해요.","input":{"request":{"requestId":"req_11cd0f34","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"<nick:5b247222>","trigger":"user-message","userMessage":"닉네임은 롤 닉네임가여?"}},"expected":{}}
|
||||
{"id":"case_79d82387","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778348415532-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_abd7b438","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_4756ecd5","nickname":"<nick:5b247222>","trigger":"user-message","userMessage":"안녕하세영"}},"expected":{}}
|
||||
{"id":"case_c810958e","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778334757877-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_6a9f1a7b","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_961ebb8c","nickname":"<nick:cfdcccbd>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_d368dec6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778322598506-0; llmOverall=3.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 확인했어요. 지금은 다 끝난 상태로 보여요.","input":{"request":{"requestId":"req_7f7d49de","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_33096f93","nickname":"<nick:3b47a030>","trigger":"user-message","userMessage":"주민신청,단통방하트인증,구인방입장,본인인증 완료햇습니다"}},"expected":{}}
|
||||
{"id":"case_f06874cb","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778447250218-0; llmOverall=1.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=좋아요, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_f2313e98","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"<nick:97543019>","trigger":"user-message","userMessage":"Okay Mr, it was pleasure to see you here"}},"expected":{}}
|
||||
{"id":"case_df73a771","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778446712145-0; llmOverall=2.25; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 맞아요. 이곳은 롤림푸스 새내기 등록을 돕는 안내 채팅입니다.","input":{"request":{"requestId":"req_a269699a","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"<nick:97543019>","trigger":"user-message","userMessage":"Yeah I guess"}},"expected":{}}
|
||||
{"id":"case_41173969","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778482229470-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse={\"message\":\"안녕하세요~\"}","input":{"request":{"requestId":"req_0ca7be91","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_3c700174","nickname":"<nick:7ecd7571>","trigger":"user-message","userMessage":"안녕하세요~"}},"expected":{}}
|
||||
{"id":"case_58a651e2","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778477393576-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_99ce06e3","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_16262d3a","nickname":"<nick:88794945>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_6b6ad52a","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778477202451-0; llmOverall=5; derivedIntent=age-question; derivedPhase=deadline-near; assistantResponse=네, 딱 27살도 가능합니다. 롤림푸스는 27세 이상만 가입이 가능해요.","input":{"request":{"requestId":"req_ed27fe73","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_16262d3a","nickname":"<nick:8e449d55>","trigger":"user-message","userMessage":"안녕하세요 혹시 딱 27살도 가능할까요?"}},"expected":{}}
|
||||
{"id":"case_87896362","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778464458764-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_c8b6b29c","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_1f854d18","nickname":"<nick:1832431a>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
{"id":"case_95b15072","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778447091058-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=아니에요, 괜찮아요.","input":{"request":{"requestId":"req_6f5881ac","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"<nick:97543019>","trigger":"user-message","userMessage":"Oh sorry sorry"}},"expected":{}}
|
||||
{"id":"case_647fb0f0","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778446903996-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요~","input":{"request":{"requestId":"req_54d027f4","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_071aecbd","nickname":"<nick:97543019>","trigger":"user-message","userMessage":"Hi everyone"}},"expected":{}}
|
||||
{"id":"case_2459a879","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1778501754586-0; llmOverall=2; derivedIntent=discord-question; derivedPhase=deadline-near; assistantResponse=아직은 어렵습니다. 디스코드 참여는 카카오톡 1단계가 먼저 완료되어야 가능해요. \n현재는 방 하트 인증이 미완료라 지금은 디코방 이용이 어려워요.","input":{"request":{"requestId":"req_ad497829","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"<nick:9f883a1e>","trigger":"user-message","userMessage":"혹시 디코방 참여해서 겜 같이해도되나요?"}},"expected":{}}
|
||||
{"id":"case_f7818487","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778501905206-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_76d6f9da","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"<nick:9f883a1e>","trigger":"user-message","userMessage":"넵"}},"expected":{}}
|
||||
{"id":"case_fe74c37e","lineNumber":0,"name":"promotion:rule-llm-disagree","notes":"promotionReason=rule-llm-disagree; sourceStreamId=1778501809468-0; llmOverall=2.5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 천천히 진행해 주세요.","input":{"request":{"requestId":"req_65762501","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"<nick:9f883a1e>","trigger":"user-message","userMessage":"넵"}},"expected":{}}
|
||||
{"id":"case_3a30d20f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778502401994-0; llmOverall=5; derivedIntent=acknowledgement; derivedPhase=deadline-near; assistantResponse=네, 완료 확인했습니다.","input":{"request":{"requestId":"req_b8dce22e","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"<nick:9f883a1e>","trigger":"user-message","userMessage":"완료요!"}},"expected":{}}
|
||||
{"id":"case_b1e0dff1","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1778501699301-0; llmOverall=5; derivedIntent=greeting; derivedPhase=deadline-near; assistantResponse=안녕하세요!","input":{"request":{"requestId":"req_d42492fe","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_a9d90912","nickname":"<nick:9f883a1e>","trigger":"user-message","userMessage":"안녕하세요"}},"expected":{}}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"<nick:greeting>","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}}
|
||||
{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","kakaotalkNickname":"프리리/95/여/D","kakaotalkNicknameVerified":true,"lastEventAt":"2026-04-16T00:10:00.000Z","photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}}
|
||||
{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","kakaotalkNickname":"프리리/95/여/다이아","kakaotalkNicknameVerified":false,"lastEventAt":"2026-04-16T00:10:00.000Z","photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}}
|
||||
|
||||
{"id":"case_119e66ab","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777693377499-0; llmOverall=1.5; derivedIntent=nickname-question; derivedPhase=deadline-near; assistantResponse=카카오톡 닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 면정학/89/남/D처럼 쓰시면 돼요.","input":{"request":{"requestId":"req_7e7337de","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"닉네임 변경 및 하트 인증은 하였는데, 혹시 어느 부분에서 문제가 있는지 여쭈어 볼 수 있을까요?"}},"expected":{}}
|
||||
{"id":"case_6bc9858f","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777692194987-0; llmOverall=1.5; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=공지방은 따로 운영하지 않습니다.","input":{"request":{"requestId":"req_c91897cf","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"죄송합니다. 제가 기억하기론 공지방이 따로 있던 것으로 기억하는데, 공지방은 따로 없는지 여쭈어 볼 수 있을까요?"}},"expected":{}}
|
||||
{"id":"case_edbec2e7","lineNumber":0,"name":"promotion:low-score","notes":"promotionReason=low-score; sourceStreamId=1777678836132-0; llmOverall=2; derivedIntent=help-where; derivedPhase=deadline-near; assistantResponse=디스코드의 주민신청 채널에서 멘션 남기면 됩니다.","input":{"request":{"requestId":"req_e69bfe09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_2096a462","nickname":"<nick:e63f2878>","trigger":"user-message","userMessage":"디코 공지 확인이랑 닉변까지 다 했는데 어디에서 멘션하면 되나여"}},"expected":{}}
|
||||
{"id":"case_756da19b","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693260088-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=아, 그러셨군요. 지금은 카카오톡 닉네임 변경만 남아 있어서 그 과정에서 막히신 거죠. \n무슨 부분이 어려우신가요? 어떤 과정에서 막히셨는지 알려 주세요.","input":{"request":{"requestId":"req_f09cd933","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"하는데"}},"expected":{}}
|
||||
{"id":"case_70b3db02","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777693242995-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=네, 배치 보려면 먼저 카카오톡 1단계부터 끝내야 해요. 현재는 닉네임 변경만 남아 있고, 마감이 임박했으니 지금 바로 닉네임만 먼저 완료해 주세요.","input":{"request":{"requestId":"req_b68c3a89","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"얼른 배치를 봐야만"}},"expected":{}}
|
||||
{"id":"case_d8efa98f","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777692242389-0; llmOverall=3.25; derivedIntent=help-how; derivedPhase=deadline-near; assistantResponse=네, 맞습니다. 구인글은 구인방에서만 올리는 게 원칙이에요.","input":{"request":{"requestId":"req_027cab09","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"user-message","userMessage":"감사합니다.\n구인방에서만 구인글을 게시하는지 마지막으로 여쭈어 볼 수 있을까요?"}},"expected":{}}
|
||||
{"id":"case_b93a3eac","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777691863103-0; llmOverall=n/a; derivedIntent=photo-question; derivedPhase=photo-rejected; assistantResponse=하트 인증은 이렇게만 다시 보내면 돼요.\n\n채팅방 오른쪽 상단 메뉴 버튼 → ‘커버 보기’ 들어가기 \n하트를 빨간색으로 바꾸기 \n그 상태를 편집 없이 원본 그대로 캡처해서 사진으로 보내기\n\n자르기나 화면 꾸미기 같은 편집은 하지 말고, 그냥 바로 스크린샷 그대로 보내주세요.","input":{"request":{"requestId":"req_1785e212","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_cc41daef","nickname":"<nick:967db24a>","trigger":"photo-failed"}},"expected":{}}
|
||||
{"id":"case_4c4e71d6","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655309842-0; llmOverall=4.75; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=무슨 말인지 모르겠어용 ㅎㅎ 한 번만 더 정확히 말해줄 수 있어임니당?","input":{"request":{"requestId":"req_1147b077","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"<nick:8a181c76>","trigger":"user-message","userMessage":"머징"}},"expected":{}}
|
||||
{"id":"case_b3ba68ec","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777655250210-0; llmOverall=n/a; derivedIntent=unknown; derivedPhase=deadline-near; assistantResponse=오키, 점수 200점 확인했어용! 잘 적어둘게요.","input":{"request":{"requestId":"req_1f66afdc","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_fc312bc0","nickname":"<nick:8a181c76>","trigger":"user-message","userMessage":"200점수따리라"}},"expected":{}}
|
||||
{"id":"case_475ee343","lineNumber":0,"name":"promotion:rare-slice","notes":"promotionReason=rare-slice; sourceStreamId=1777649519625-0; llmOverall=4.75; derivedIntent=acknowledgement; derivedPhase=completed; assistantResponse=네, 다 잘 끝났으니까 푹 쉬었다 다녀오세요~","input":{"request":{"requestId":"req_11add9b0","skill":"onboarding","chatroomId":"room_0575da24","userId":"user_57a7acdc","nickname":"<nick:484b87ad>","trigger":"user-message","userMessage":"가겠습니당~~"}},"expected":{}}
|
||||
@@ -0,0 +1,202 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import { judgeProcessRule } from "./process-judge.js";
|
||||
|
||||
function makeTrace(
|
||||
overrides?: Partial<ProcessTraceSnapshot["steps"]>,
|
||||
): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
classify: {
|
||||
name: "classify",
|
||||
startedAtMs: 1,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
intent: "status-question",
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
},
|
||||
derive: {
|
||||
name: "derive",
|
||||
startedAtMs: 2,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm-with-tools",
|
||||
shouldRespond: true,
|
||||
tier: 1,
|
||||
},
|
||||
},
|
||||
plan: {
|
||||
name: "plan",
|
||||
startedAtMs: 3,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
approach: "validate-then-explain",
|
||||
mustAvoid: [],
|
||||
mustInclude: ["남은 절차"],
|
||||
suggestedTools: ["validate-nickname"],
|
||||
targetLengthChars: 120,
|
||||
},
|
||||
},
|
||||
toolCalls: [
|
||||
{
|
||||
name: "tool-call",
|
||||
startedAtMs: 4,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
name: "validate-nickname",
|
||||
args: {
|
||||
nickname: "프리리/95/여/D",
|
||||
},
|
||||
ok: true,
|
||||
resultSummary: '{"valid":true}',
|
||||
},
|
||||
},
|
||||
],
|
||||
respond: {
|
||||
name: "respond",
|
||||
startedAtMs: 5,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
draftText: "방 하트 인증만 남았어요.",
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
},
|
||||
},
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 6,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "pass",
|
||||
reasons: [],
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("judgeProcessRule", () => {
|
||||
it("passes active process expectations", () => {
|
||||
const result = judgeProcessRule(
|
||||
{
|
||||
intent: "status-question",
|
||||
processExpected: {
|
||||
expectedApproach: "validate-then-explain",
|
||||
expectedTools: ["validate-nickname"],
|
||||
minSteps: 5,
|
||||
},
|
||||
},
|
||||
makeTrace(),
|
||||
);
|
||||
|
||||
expect(result.score).toBe(1);
|
||||
expect(result.totalCount).toBeGreaterThan(0);
|
||||
expect(result.checks.every((check) => check.skipped || check.passed)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("fails intent, tool, and approach mismatches", () => {
|
||||
const result = judgeProcessRule(
|
||||
{
|
||||
intent: "photo-question",
|
||||
processExpected: {
|
||||
expectedApproach: "answer-direct",
|
||||
expectedTools: ["check-onboarding-progress"],
|
||||
toolMode: "exact",
|
||||
},
|
||||
},
|
||||
makeTrace(),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "intent-match")?.passed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "planner-approach")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tools-selected-correctly")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("fails invalid tool arguments", () => {
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
makeTrace({
|
||||
toolCalls: [
|
||||
{
|
||||
name: "tool-call",
|
||||
startedAtMs: 4,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
name: "validate-nickname",
|
||||
args: {},
|
||||
ok: false,
|
||||
resultSummary: "invalid",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tool-args-valid")?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("fails tier and validator process regressions", () => {
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
makeTrace({
|
||||
derive: {
|
||||
name: "derive",
|
||||
startedAtMs: 2,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm-with-tools",
|
||||
shouldRespond: true,
|
||||
tier: 0,
|
||||
},
|
||||
},
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 6,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "reject",
|
||||
reasons: ["stateConsistency"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tier-appropriate")?.passed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "validate-no-reject")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("skips optional expectations when a golden case has not opted in", () => {
|
||||
const result = judgeProcessRule({}, makeTrace());
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "intent-match")?.skipped,
|
||||
).toBe(true);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "planner-approach")
|
||||
?.skipped,
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,353 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import {
|
||||
getEvalProcessLlmJudgeMaxTokens,
|
||||
getLlmJudgeModel,
|
||||
} from "../../config.js";
|
||||
import type {
|
||||
EvalCaseExpected,
|
||||
ProcessJudgeCheck,
|
||||
ProcessJudgeResult,
|
||||
} from "../types.js";
|
||||
|
||||
const processLlmJudgeSchema = z.object({
|
||||
rationale: z.string().trim().min(1).max(300),
|
||||
score: z.number().min(0).max(1),
|
||||
});
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function makeCheck(
|
||||
step: string,
|
||||
rule: ProcessJudgeCheck["rule"],
|
||||
passed: boolean,
|
||||
detail?: string,
|
||||
): ProcessJudgeCheck {
|
||||
return {
|
||||
step,
|
||||
rule,
|
||||
passed,
|
||||
skipped: false,
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSkippedCheck(
|
||||
step: string,
|
||||
rule: ProcessJudgeCheck["rule"],
|
||||
detail: string,
|
||||
): ProcessJudgeCheck {
|
||||
return {
|
||||
step,
|
||||
rule,
|
||||
passed: true,
|
||||
skipped: true,
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudgeResult {
|
||||
const activeChecks = checks.filter((check) => !check.skipped);
|
||||
const passedCount = activeChecks.filter((check) => check.passed).length;
|
||||
const totalCount = activeChecks.length;
|
||||
const result: ProcessJudgeResult = {
|
||||
checks,
|
||||
passedCount,
|
||||
score: totalCount === 0 ? 1 : round(passedCount / totalCount),
|
||||
totalCount,
|
||||
};
|
||||
|
||||
if (rationale) {
|
||||
result.rationale = rationale;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function hasAllTools(
|
||||
actualTools: string[],
|
||||
expectedTools: string[],
|
||||
mode: "exact" | "subset",
|
||||
): boolean {
|
||||
const actual = new Set(actualTools);
|
||||
const expected = new Set(expectedTools);
|
||||
const subset = expectedTools.every((tool) => actual.has(tool));
|
||||
|
||||
if (mode === "subset") {
|
||||
return subset;
|
||||
}
|
||||
|
||||
return subset && actual.size === expected.size;
|
||||
}
|
||||
|
||||
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
||||
const scalarSteps = [
|
||||
trace.steps.classify,
|
||||
trace.steps.derive,
|
||||
trace.steps.plan,
|
||||
trace.steps.respond,
|
||||
trace.steps.critic,
|
||||
trace.steps.validate,
|
||||
].filter(Boolean).length;
|
||||
|
||||
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
||||
}
|
||||
|
||||
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
||||
const toolCalls = trace.steps.toolCalls ?? [];
|
||||
if (toolCalls.length === 0) {
|
||||
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
||||
}
|
||||
|
||||
const registry = createBuiltInToolRegistry();
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const call of toolCalls) {
|
||||
try {
|
||||
registry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||
} catch {
|
||||
failures.push(call.output.name);
|
||||
}
|
||||
}
|
||||
|
||||
return makeCheck(
|
||||
"tool-call",
|
||||
"tool-args-valid",
|
||||
failures.length === 0,
|
||||
failures.length === 0
|
||||
? undefined
|
||||
: `invalid args for: ${failures.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function judgeProcessRule(
|
||||
expected: EvalCaseExpected,
|
||||
trace: ProcessTraceSnapshot,
|
||||
): ProcessJudgeResult {
|
||||
const checks: ProcessJudgeCheck[] = [];
|
||||
|
||||
if (expected.intent) {
|
||||
const actualIntent = trace.steps.classify?.output.intent;
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"classify",
|
||||
"intent-match",
|
||||
actualIntent === expected.intent,
|
||||
`expected=${expected.intent}, actual=${actualIntent ?? "missing"}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(makeSkippedCheck("classify", "intent-match", "no expected intent"));
|
||||
}
|
||||
|
||||
const derived = trace.steps.derive?.output;
|
||||
if (derived?.responseMode === "llm-with-tools") {
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"derive",
|
||||
"tier-appropriate",
|
||||
derived.tier >= 1,
|
||||
`responseMode=${derived.responseMode}, tier=${derived.tier}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck(
|
||||
"derive",
|
||||
"tier-appropriate",
|
||||
"responseMode is not llm-with-tools",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const processExpected = expected.processExpected;
|
||||
if (processExpected?.expectedApproach) {
|
||||
const actualApproach = trace.steps.plan?.output.approach;
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"plan",
|
||||
"planner-approach",
|
||||
actualApproach === processExpected.expectedApproach,
|
||||
`expected=${processExpected.expectedApproach}, actual=${actualApproach ?? "missing"}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck("plan", "planner-approach", "no expected approach"),
|
||||
);
|
||||
}
|
||||
|
||||
if (processExpected?.expectedTools?.length) {
|
||||
const actualTools =
|
||||
trace.steps.toolCalls?.map((call) => call.output.name) ?? [];
|
||||
const mode = processExpected.toolMode ?? "subset";
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"tool-call",
|
||||
"tools-selected-correctly",
|
||||
hasAllTools(actualTools, processExpected.expectedTools, mode),
|
||||
`expected=${processExpected.expectedTools.join(",")}, actual=${actualTools.join(",")}, mode=${mode}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck(
|
||||
"tool-call",
|
||||
"tools-selected-correctly",
|
||||
"no expected tools",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
checks.push(validateToolArgs(trace));
|
||||
|
||||
if (processExpected?.minSteps !== undefined) {
|
||||
const actualSteps = countRecordedSteps(trace);
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"trace",
|
||||
"min-steps",
|
||||
actualSteps >= processExpected.minSteps,
|
||||
`expected>=${processExpected.minSteps}, actual=${actualSteps}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(makeSkippedCheck("trace", "min-steps", "no minSteps"));
|
||||
}
|
||||
|
||||
const critic = trace.steps.critic?.output;
|
||||
if (critic && derived?.tier === 2) {
|
||||
const validationStatus = trace.steps.validate?.output.status;
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"critic",
|
||||
"critic-decision-justified",
|
||||
critic.decision !== "keep" || validationStatus !== "reject",
|
||||
`decision=${critic.decision}, validate=${validationStatus ?? "missing"}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck(
|
||||
"critic",
|
||||
"critic-decision-justified",
|
||||
"no tier 2 critic decision",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const validate = trace.steps.validate?.output;
|
||||
if (validate) {
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"validate",
|
||||
"validate-no-reject",
|
||||
validate.status !== "reject",
|
||||
`status=${validate.status}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeCheck("validate", "validate-no-reject", false, "missing validate step"),
|
||||
);
|
||||
}
|
||||
|
||||
return finalize(checks);
|
||||
}
|
||||
|
||||
function buildProcessJudgePrompt(
|
||||
trace: ProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): string {
|
||||
return [
|
||||
"온보딩 에이전트의 단계별 실행 trace를 평가해 주세요.",
|
||||
"planner 선택, tool 선택/인자, critic 판단, 최종 검증 흐름이 합리적인지 보세요.",
|
||||
"score는 0부터 1 사이 숫자입니다.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(options.request),
|
||||
"",
|
||||
"[trace]",
|
||||
JSON.stringify(trace),
|
||||
"",
|
||||
"[finalText]",
|
||||
options.finalText,
|
||||
"",
|
||||
'JSON object only: {"score":0.8,"rationale":"..."}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function judgeProcessWithLlm(
|
||||
llmClient: LlmClient,
|
||||
trace: ProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): Promise<ProcessJudgeResult> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
return finalize(
|
||||
[
|
||||
makeSkippedCheck(
|
||||
"trace",
|
||||
"validate-no-reject",
|
||||
"no LLM endpoint for process judge",
|
||||
),
|
||||
],
|
||||
"skipped: no LLM endpoint",
|
||||
);
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 에이전트의 과정 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildProcessJudgePrompt(trace, options),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: getEvalProcessLlmJudgeMaxTokens(),
|
||||
modelOverride: getLlmJudgeModel(),
|
||||
role: "critic",
|
||||
temperature: 0,
|
||||
tier: 2,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
return finalize(
|
||||
[makeCheck("trace", "validate-no-reject", false, "judge call failed")],
|
||||
"process judge call failed",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = processLlmJudgeSchema.parse(JSON.parse(response.content));
|
||||
const passed = parsed.score >= 0.7;
|
||||
return finalize(
|
||||
[
|
||||
makeCheck(
|
||||
"trace",
|
||||
"validate-no-reject",
|
||||
passed,
|
||||
`llmScore=${round(parsed.score)}`,
|
||||
),
|
||||
],
|
||||
parsed.rationale,
|
||||
);
|
||||
} catch {
|
||||
return finalize(
|
||||
[makeCheck("trace", "validate-no-reject", false, "invalid judge json")],
|
||||
"invalid process judge json",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { createEloRating, scoreForVerdict, updateEloPair } from "./elo.js";
|
||||
|
||||
describe("elo", () => {
|
||||
it("updates equal ratings by the standard K=32 margin", () => {
|
||||
const control = createEloRating("control");
|
||||
const variant = createEloRating("variant");
|
||||
|
||||
const updated = updateEloPair(control, variant, 1);
|
||||
|
||||
expect(updated.ratingA.elo).toBe(1516);
|
||||
expect(updated.ratingB.elo).toBe(1484);
|
||||
expect(updated.ratingA.matches).toBe(1);
|
||||
expect(updated.ratingA.wins).toBe(1);
|
||||
});
|
||||
|
||||
it("maps side-by-side verdicts to scores", () => {
|
||||
expect(scoreForVerdict("A")).toBe(1);
|
||||
expect(scoreForVerdict("B")).toBe(0);
|
||||
expect(scoreForVerdict("tie")).toBe(0.5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
export interface EloRating {
|
||||
id: string;
|
||||
elo: number;
|
||||
matches: number;
|
||||
wins: number;
|
||||
}
|
||||
|
||||
export interface EloUpdate {
|
||||
ratingA: EloRating;
|
||||
ratingB: EloRating;
|
||||
}
|
||||
|
||||
const DEFAULT_ELO = 1_500;
|
||||
const DEFAULT_K_FACTOR = 32;
|
||||
|
||||
function expectedScore(left: number, right: number): number {
|
||||
return 1 / (1 + 10 ** ((right - left) / 400));
|
||||
}
|
||||
|
||||
function roundElo(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
export function createEloRating(id: string): EloRating {
|
||||
return {
|
||||
id,
|
||||
elo: DEFAULT_ELO,
|
||||
matches: 0,
|
||||
wins: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function updateEloPair(
|
||||
ratingA: EloRating,
|
||||
ratingB: EloRating,
|
||||
scoreA: 0 | 0.5 | 1,
|
||||
kFactor = DEFAULT_K_FACTOR,
|
||||
): EloUpdate {
|
||||
const expectedA = expectedScore(ratingA.elo, ratingB.elo);
|
||||
const scoreB = (1 - scoreA) as 0 | 0.5 | 1;
|
||||
const expectedB = 1 - expectedA;
|
||||
|
||||
return {
|
||||
ratingA: {
|
||||
...ratingA,
|
||||
elo: roundElo(ratingA.elo + kFactor * (scoreA - expectedA)),
|
||||
matches: ratingA.matches + 1,
|
||||
wins: ratingA.wins + (scoreA === 1 ? 1 : 0),
|
||||
},
|
||||
ratingB: {
|
||||
...ratingB,
|
||||
elo: roundElo(ratingB.elo + kFactor * (scoreB - expectedB)),
|
||||
matches: ratingB.matches + 1,
|
||||
wins: ratingB.wins + (scoreB === 1 ? 1 : 0),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function scoreForVerdict(
|
||||
verdict: "A" | "B" | "tie",
|
||||
): 0 | 0.5 | 1 {
|
||||
if (verdict === "A") {
|
||||
return 1;
|
||||
}
|
||||
if (verdict === "B") {
|
||||
return 0;
|
||||
}
|
||||
return 0.5;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { EvalArchiveEntry } from "../archive.js";
|
||||
import { buildCurrentPromptSnapshot, proposeVariants } from "./propose.js";
|
||||
|
||||
function archiveEntry(): EvalArchiveEntry {
|
||||
return {
|
||||
streamId: "1-0",
|
||||
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "사진 어디로 보내요?",
|
||||
},
|
||||
response: {
|
||||
requestId: "req-1",
|
||||
text: "디스코드에 보내주세요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "photo-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "디스코드에 보내주세요.",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: 4,
|
||||
accuracy: 1,
|
||||
concision: 4,
|
||||
scope: 2,
|
||||
overall: 2.75,
|
||||
},
|
||||
rationale: "wrong channel",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("proposeVariants", () => {
|
||||
it("parses LLM JSON into stable prompt and tool proposals", async () => {
|
||||
const llmClient = {
|
||||
hasEndpoints: () => true,
|
||||
chatCompletion: () =>
|
||||
Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
prompts: [
|
||||
{
|
||||
description: "Clarify photo location",
|
||||
diff: [
|
||||
{
|
||||
layer: "policy",
|
||||
op: "append",
|
||||
text: "사진 인증 위치를 디스코드와 혼동하지 마.",
|
||||
},
|
||||
],
|
||||
rationale: "low-score cases confuse photo location",
|
||||
expectedImprovement: "fewer wrong-channel answers",
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
description: "Photo routing helper",
|
||||
toolName: "lookup-photo-route",
|
||||
schema: { type: "object" },
|
||||
whyNeeded: "Repeated hallucination on photo channel",
|
||||
},
|
||||
],
|
||||
}),
|
||||
endpoint: "mock",
|
||||
message: { role: "assistant", content: "{}" },
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
}),
|
||||
} as unknown as LlmClient;
|
||||
|
||||
const proposals = await proposeVariants(llmClient, {
|
||||
slice: {
|
||||
tier: 0,
|
||||
intent: "photo-question",
|
||||
phase: "photo-pending",
|
||||
},
|
||||
recentLabels: [],
|
||||
worstCases: [archiveEntry()],
|
||||
currentPromptSnapshot: buildCurrentPromptSnapshot(
|
||||
{
|
||||
tier: 0,
|
||||
intent: "photo-question",
|
||||
phase: "photo-pending",
|
||||
},
|
||||
archiveEntry(),
|
||||
),
|
||||
});
|
||||
|
||||
expect(proposals.prompts).toHaveLength(1);
|
||||
expect(proposals.prompts[0]?.variantId).toMatch(/^[a-f0-9]{8}$/u);
|
||||
expect(proposals.prompts[0]?.diff[0]).toMatchObject({
|
||||
layer: "policy",
|
||||
op: "append",
|
||||
});
|
||||
expect(proposals.tools[0]).toMatchObject({
|
||||
toolName: "lookup-photo-route",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,300 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import {
|
||||
PromptBuilder,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@skills/onboarding/prompts/builder";
|
||||
import {
|
||||
getEvalTournamentMinCases,
|
||||
getLlmProposeMaxTokens,
|
||||
getLlmProposeModel,
|
||||
} from "../../config.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||
import { getEvalArchiveCollection } from "../archive.js";
|
||||
import type { SliceKey } from "../aggregator.js";
|
||||
import { queryArchivedReviewLabels } from "../review/label-archive.js";
|
||||
import type { ReviewLabel } from "../review/queue.js";
|
||||
|
||||
export interface ProposalInput {
|
||||
slice: SliceKey;
|
||||
recentLabels: ReviewLabel[];
|
||||
worstCases: EvalArchiveEntry[];
|
||||
currentPromptSnapshot: string;
|
||||
}
|
||||
|
||||
export interface PromptVariantProposal extends PromptVariantManifest {
|
||||
description: string;
|
||||
expectedImprovement: string;
|
||||
rationale: string;
|
||||
}
|
||||
|
||||
export interface ToolProposal {
|
||||
description: string;
|
||||
schema: unknown;
|
||||
toolName: string;
|
||||
variantId: string;
|
||||
whyNeeded: string;
|
||||
}
|
||||
|
||||
const promptDiffSchema = z.object({
|
||||
layer: z.enum([
|
||||
"context",
|
||||
"examples",
|
||||
"persona",
|
||||
"policy",
|
||||
"state",
|
||||
"tool-policy",
|
||||
]),
|
||||
op: z.enum(["append", "remove", "replace"]),
|
||||
reason: z.string().trim().max(500).optional(),
|
||||
text: z.string().trim().max(4_000).optional(),
|
||||
});
|
||||
|
||||
const proposalSchema = z.object({
|
||||
prompts: z
|
||||
.array(
|
||||
z.object({
|
||||
description: z.string().trim().min(1).max(300),
|
||||
diff: z.array(promptDiffSchema).min(1).max(10),
|
||||
expectedImprovement: z.string().trim().min(1).max(500),
|
||||
rationale: z.string().trim().min(1).max(500),
|
||||
}),
|
||||
)
|
||||
.max(5),
|
||||
tools: z
|
||||
.array(
|
||||
z.object({
|
||||
description: z.string().trim().min(1).max(300),
|
||||
schema: z.unknown(),
|
||||
toolName: z.string().trim().min(1).max(80),
|
||||
whyNeeded: z.string().trim().min(1).max(500),
|
||||
}),
|
||||
)
|
||||
.max(5)
|
||||
.default([]),
|
||||
});
|
||||
|
||||
function stableHash(value: unknown): string {
|
||||
return createHash("sha256")
|
||||
.update(JSON.stringify(value))
|
||||
.digest("hex")
|
||||
.slice(0, 8);
|
||||
}
|
||||
|
||||
function normalizePromptDiff(diff: z.infer<typeof promptDiffSchema>): PromptLayerDiff {
|
||||
const normalized: PromptLayerDiff = {
|
||||
layer: diff.layer as PromptLayerName,
|
||||
op: diff.op,
|
||||
};
|
||||
if (diff.reason !== undefined) {
|
||||
normalized.reason = diff.reason;
|
||||
}
|
||||
if (diff.text !== undefined) {
|
||||
normalized.text = diff.text;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: { kind: slice.intent } as Intent,
|
||||
phase: { kind: slice.phase } as OnboardingPhase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 384,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPromptContext(): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
export function buildCurrentPromptSnapshot(
|
||||
slice: SliceKey,
|
||||
archiveEntry?: EvalArchiveEntry,
|
||||
): string {
|
||||
return PromptBuilder.forDerived(
|
||||
archiveEntry?.derived ?? derivedFromSlice(slice),
|
||||
null,
|
||||
defaultPromptContext(),
|
||||
).buildSystemPrompt();
|
||||
}
|
||||
|
||||
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
|
||||
return {
|
||||
streamId: entry.streamId,
|
||||
request: entry.request,
|
||||
response: entry.response.text,
|
||||
validation: entry.validation,
|
||||
llmJudge: entry.llmJudge,
|
||||
processJudge: entry.processJudge,
|
||||
process: entry.process,
|
||||
evaluatedAt: entry.evaluatedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function buildProposalPrompt(input: ProposalInput): string {
|
||||
return [
|
||||
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
|
||||
"반드시 JSON object만 출력하세요.",
|
||||
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
|
||||
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
|
||||
"",
|
||||
"[slice]",
|
||||
JSON.stringify(input.slice),
|
||||
"",
|
||||
"[recent_human_labels]",
|
||||
JSON.stringify(input.recentLabels.slice(0, 20)),
|
||||
"",
|
||||
"[worst_cases]",
|
||||
JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)),
|
||||
"",
|
||||
"[current_prompt_snapshot]",
|
||||
input.currentPromptSnapshot,
|
||||
"",
|
||||
'JSON schema: {"prompts":[{"description":"...","diff":[{"layer":"policy","op":"append","text":"...","reason":"..."}],"rationale":"...","expectedImprovement":"..."}],"tools":[{"description":"...","toolName":"...","schema":{},"whyNeeded":"..."}]}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function proposeVariants(
|
||||
llmClient: LlmClient,
|
||||
input: ProposalInput,
|
||||
): Promise<{ prompts: PromptVariantProposal[]; tools: ToolProposal[] }> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
throw new Error("Cannot propose variants without an LLM endpoint");
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildProposalPrompt(input),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: getLlmProposeMaxTokens(),
|
||||
modelOverride: getLlmProposeModel(),
|
||||
temperature: 0.2,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
throw new Error("Proposal LLM call failed");
|
||||
}
|
||||
|
||||
const parsed = proposalSchema.parse(JSON.parse(response.content));
|
||||
const prompts = parsed.prompts.map((proposal) => {
|
||||
const diff = proposal.diff.map(normalizePromptDiff);
|
||||
return {
|
||||
...proposal,
|
||||
diff,
|
||||
variantId: stableHash({
|
||||
description: proposal.description,
|
||||
diff,
|
||||
rationale: proposal.rationale,
|
||||
}),
|
||||
};
|
||||
});
|
||||
const tools = parsed.tools.map((proposal) => ({
|
||||
...proposal,
|
||||
variantId: stableHash(proposal),
|
||||
}));
|
||||
|
||||
return {
|
||||
prompts,
|
||||
tools,
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildProposalInputFromArchive(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
slice: SliceKey,
|
||||
options?: {
|
||||
force?: boolean | undefined;
|
||||
limit?: number | undefined;
|
||||
minCases?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
},
|
||||
): Promise<ProposalInput> {
|
||||
const limit = options?.limit ?? 50;
|
||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
tier: slice.tier,
|
||||
"derived.intent.kind": slice.intent,
|
||||
"derived.phase.kind": slice.phase,
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
const minCases = options?.minCases ?? getEvalTournamentMinCases();
|
||||
|
||||
if (!options?.force && entries.length < minCases) {
|
||||
throw new Error(
|
||||
`Not enough archived cases for propose: ${entries.length}/${minCases}. Use --force to override.`,
|
||||
);
|
||||
}
|
||||
|
||||
const ids = new Set(entries.map((entry) => entry.streamId));
|
||||
const recentLabels = (await queryArchivedReviewLabels(mongoConnection, {
|
||||
limit: 1_000,
|
||||
nowMs: options?.nowMs,
|
||||
})).filter((label) => ids.has(label.evalStreamId));
|
||||
const worstCases = [...entries]
|
||||
.filter((entry) => getLlmOverall(entry) !== null)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
(getLlmOverall(left) ?? Number.POSITIVE_INFINITY) -
|
||||
(getLlmOverall(right) ?? Number.POSITIVE_INFINITY),
|
||||
)
|
||||
.slice(0, 5);
|
||||
|
||||
return {
|
||||
slice,
|
||||
recentLabels,
|
||||
worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5),
|
||||
currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]),
|
||||
};
|
||||
}
|
||||
|
||||
export async function writePromptVariantProposal(
|
||||
proposal: PromptVariantProposal,
|
||||
outputDir = "src/eval/proposals",
|
||||
): Promise<string> {
|
||||
const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`);
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(`${outputPath}`, `${JSON.stringify(proposal, null, 2)}\n`, "utf8");
|
||||
return outputPath;
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { EvalArchiveEntry } from "../archive.js";
|
||||
import { runTournament } from "./tournament.js";
|
||||
|
||||
function archiveEntry(streamId: string): EvalArchiveEntry {
|
||||
return {
|
||||
streamId,
|
||||
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
request: {
|
||||
requestId: streamId,
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "사진 어디로 보내요?",
|
||||
},
|
||||
response: {
|
||||
requestId: streamId,
|
||||
text: "이 채팅방에 올려주세요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "photo-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "이 채팅방에 올려주세요.",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: 4,
|
||||
accuracy: 4,
|
||||
concision: 4,
|
||||
scope: 4,
|
||||
overall: 4,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("runTournament", () => {
|
||||
it("runs swapped A/B matches and updates ELO toward winning variants", async () => {
|
||||
const llmClient = {
|
||||
hasEndpoints: () => true,
|
||||
chatCompletion: (
|
||||
messages: Array<{ content: string; role: string }>,
|
||||
options?: { jsonMode?: boolean | undefined },
|
||||
) => {
|
||||
const content = messages.map((message) => message.content).join("\n");
|
||||
if (options?.jsonMode) {
|
||||
const verdict = content.includes("[response_A]\nvariant answer")
|
||||
? "A"
|
||||
: "B";
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
verdict,
|
||||
rationale: "variant is more accurate",
|
||||
}),
|
||||
endpoint: "mock",
|
||||
message: { role: "assistant", content: "{}" },
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: content.includes("사진 인증 위치를 명시해")
|
||||
? "variant answer"
|
||||
: "control answer",
|
||||
endpoint: "mock",
|
||||
message: { role: "assistant", content: "ok" },
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
},
|
||||
} as unknown as LlmClient;
|
||||
|
||||
const result = await runTournament(llmClient, {
|
||||
cases: [archiveEntry("1-0"), archiveEntry("2-0")],
|
||||
maxMatches: 4,
|
||||
nowMs: Date.parse("2026-05-01T02:00:00.000Z"),
|
||||
variantIds: ["control", "abc12345"],
|
||||
variantManifests: {
|
||||
abc12345: {
|
||||
variantId: "abc12345",
|
||||
description: "photo fix",
|
||||
diff: [
|
||||
{
|
||||
layer: "policy",
|
||||
op: "append",
|
||||
text: "사진 인증 위치를 명시해.",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.generatedAt).toBe("2026-05-01T02:00:00.000Z");
|
||||
expect(result.matches).toHaveLength(4);
|
||||
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
|
||||
"control",
|
||||
"abc12345",
|
||||
]);
|
||||
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
|
||||
"abc12345",
|
||||
"control",
|
||||
]);
|
||||
expect(result.variants[0]).toMatchObject({
|
||||
id: "abc12345",
|
||||
matches: 4,
|
||||
wins: 4,
|
||||
});
|
||||
expect(result.variants[0]?.elo).toBeGreaterThan(1_500);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,363 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import { getEvalTournamentMaxMatches } from "../../config.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||
import { getEvalArchiveCollection } from "../archive.js";
|
||||
import {
|
||||
createEloRating,
|
||||
scoreForVerdict,
|
||||
updateEloPair,
|
||||
type EloRating,
|
||||
} from "./elo.js";
|
||||
|
||||
export const EVAL_TOURNAMENT_COLLECTION = "eval_tournaments";
|
||||
|
||||
export interface TournamentMatch {
|
||||
caseId: string;
|
||||
judgeRationale: string;
|
||||
judgeVerdict: "A" | "B" | "tie";
|
||||
matchId: string;
|
||||
responseA: string;
|
||||
responseB: string;
|
||||
variantA: string;
|
||||
variantB: string;
|
||||
}
|
||||
|
||||
export interface TournamentResult {
|
||||
generatedAt: string;
|
||||
matches: TournamentMatch[];
|
||||
variants: EloRating[];
|
||||
}
|
||||
|
||||
interface TournamentCollectionLike {
|
||||
createIndex(
|
||||
keys: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
): Promise<unknown>;
|
||||
find(filter: Record<string, unknown>): {
|
||||
limit(value: number): {
|
||||
sort(value: Record<string, 1 | -1>): {
|
||||
toArray(): Promise<TournamentResult[]>;
|
||||
};
|
||||
};
|
||||
sort(value: Record<string, 1 | -1>): {
|
||||
limit(value: number): {
|
||||
toArray(): Promise<TournamentResult[]>;
|
||||
};
|
||||
};
|
||||
};
|
||||
insertOne(document: TournamentResult): Promise<unknown>;
|
||||
}
|
||||
|
||||
const sideBySideJudgeSchema = z.object({
|
||||
rationale: z.string().trim().min(1).max(300),
|
||||
verdict: z.enum(["A", "B", "tie"]),
|
||||
});
|
||||
|
||||
function stableHash(value: unknown): string {
|
||||
return createHash("sha256")
|
||||
.update(JSON.stringify(value))
|
||||
.digest("hex")
|
||||
.slice(0, 12);
|
||||
}
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
function getVariantManifest(
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): PromptVariantManifest | string | null {
|
||||
if (variantId === "control") {
|
||||
return null;
|
||||
}
|
||||
return manifests?.[variantId] ?? variantId;
|
||||
}
|
||||
|
||||
async function generateTournamentResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
let builder = PromptBuilder.forDerived(
|
||||
entry.derived,
|
||||
null,
|
||||
promptContextForCase(entry),
|
||||
);
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (manifest) {
|
||||
builder = builder.applyVariant(manifest);
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(builder.build(), {
|
||||
maxTokens: entry.derived.suggestedMaxTokens,
|
||||
temperature: 0,
|
||||
tier: entry.derived.tier,
|
||||
});
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
entry: EvalArchiveEntry,
|
||||
responseA: string,
|
||||
responseB: string,
|
||||
): string {
|
||||
return [
|
||||
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
|
||||
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
|
||||
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(entry.request),
|
||||
"",
|
||||
"[derived]",
|
||||
JSON.stringify(entry.derived),
|
||||
"",
|
||||
"[archived_reference_response]",
|
||||
entry.response.text,
|
||||
"",
|
||||
"[response_A]",
|
||||
responseA,
|
||||
"",
|
||||
"[response_B]",
|
||||
responseB,
|
||||
"",
|
||||
'JSON object only: {"verdict":"A","rationale":"..."}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function judgeTournamentMatch(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
responseA: string,
|
||||
responseB: string,
|
||||
): Promise<{ rationale: string; verdict: "A" | "B" | "tie" }> {
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildJudgePrompt(entry, responseA, responseB),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: 256,
|
||||
temperature: 0,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
return {
|
||||
rationale: "judge-call-failed",
|
||||
verdict: "tie",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return sideBySideJudgeSchema.parse(JSON.parse(response.content));
|
||||
} catch {
|
||||
return {
|
||||
rationale: "invalid-judge-json",
|
||||
verdict: "tie",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function getTournamentCollection(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
): TournamentCollectionLike {
|
||||
return mongoConnection.collection(
|
||||
EVAL_TOURNAMENT_COLLECTION,
|
||||
) as TournamentCollectionLike;
|
||||
}
|
||||
|
||||
async function ensureTournamentIndexes(
|
||||
collection: TournamentCollectionLike,
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ generatedAt: -1 }),
|
||||
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
|
||||
]);
|
||||
}
|
||||
|
||||
function buildVariantPairs(variantIds: string[]): Array<[string, string]> {
|
||||
const pairs: Array<[string, string]> = [];
|
||||
for (let left = 0; left < variantIds.length; left++) {
|
||||
for (let right = left + 1; right < variantIds.length; right++) {
|
||||
const variantA = variantIds[left];
|
||||
const variantB = variantIds[right];
|
||||
if (variantA && variantB) {
|
||||
pairs.push([variantA, variantB], [variantB, variantA]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
export async function runTournament(
|
||||
llmClient: LlmClient,
|
||||
options: {
|
||||
cases: EvalArchiveEntry[];
|
||||
maxMatches?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
variantIds: string[];
|
||||
variantManifests?: Record<string, PromptVariantManifest> | undefined;
|
||||
},
|
||||
): Promise<TournamentResult> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
throw new Error("Cannot run tournament without an LLM endpoint");
|
||||
}
|
||||
if (options.variantIds.length < 2) {
|
||||
throw new Error("Tournament requires at least two variants");
|
||||
}
|
||||
|
||||
const maxMatches = options.maxMatches ?? getEvalTournamentMaxMatches();
|
||||
const ratings = new Map(
|
||||
options.variantIds.map((variantId) => [variantId, createEloRating(variantId)]),
|
||||
);
|
||||
const matches: TournamentMatch[] = [];
|
||||
const pairs = buildVariantPairs(options.variantIds);
|
||||
|
||||
for (const entry of options.cases) {
|
||||
for (const [variantA, variantB] of pairs) {
|
||||
if (matches.length >= maxMatches) {
|
||||
break;
|
||||
}
|
||||
|
||||
const [responseA, responseB] = await Promise.all([
|
||||
generateTournamentResponse(
|
||||
llmClient,
|
||||
entry,
|
||||
variantA,
|
||||
options.variantManifests,
|
||||
),
|
||||
generateTournamentResponse(
|
||||
llmClient,
|
||||
entry,
|
||||
variantB,
|
||||
options.variantManifests,
|
||||
),
|
||||
]);
|
||||
const judge = await judgeTournamentMatch(
|
||||
llmClient,
|
||||
entry,
|
||||
responseA,
|
||||
responseB,
|
||||
);
|
||||
const ratingA = ratings.get(variantA);
|
||||
const ratingB = ratings.get(variantB);
|
||||
if (!ratingA || !ratingB) {
|
||||
throw new Error("Missing tournament rating");
|
||||
}
|
||||
const updated = updateEloPair(
|
||||
ratingA,
|
||||
ratingB,
|
||||
scoreForVerdict(judge.verdict),
|
||||
);
|
||||
ratings.set(variantA, updated.ratingA);
|
||||
ratings.set(variantB, updated.ratingB);
|
||||
|
||||
matches.push({
|
||||
caseId: entry.streamId,
|
||||
judgeRationale: judge.rationale,
|
||||
judgeVerdict: judge.verdict,
|
||||
matchId: stableHash({
|
||||
caseId: entry.streamId,
|
||||
index: matches.length,
|
||||
variantA,
|
||||
variantB,
|
||||
}),
|
||||
responseA,
|
||||
responseB,
|
||||
variantA,
|
||||
variantB,
|
||||
});
|
||||
}
|
||||
if (matches.length >= maxMatches) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
|
||||
matches,
|
||||
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadTournamentCases(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options: {
|
||||
caseIds?: string[] | undefined;
|
||||
limit?: number | undefined;
|
||||
},
|
||||
): Promise<EvalArchiveEntry[]> {
|
||||
const filter =
|
||||
options.caseIds && options.caseIds.length > 0
|
||||
? {
|
||||
streamId: {
|
||||
$in: options.caseIds,
|
||||
},
|
||||
}
|
||||
: {};
|
||||
|
||||
return getEvalArchiveCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(options.limit ?? getEvalTournamentMaxMatches())
|
||||
.toArray();
|
||||
}
|
||||
|
||||
export async function saveTournamentResult(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
result: TournamentResult,
|
||||
): Promise<void> {
|
||||
const collection = getTournamentCollection(mongoConnection);
|
||||
await ensureTournamentIndexes(collection);
|
||||
await collection.insertOne(result);
|
||||
}
|
||||
|
||||
export async function queryTournamentResults(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
variantId?: string | undefined;
|
||||
},
|
||||
): Promise<TournamentResult[]> {
|
||||
const filter = options?.variantId
|
||||
? {
|
||||
"variants.id": options.variantId,
|
||||
}
|
||||
: {};
|
||||
|
||||
return getTournamentCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ generatedAt: -1 })
|
||||
.limit(options?.limit ?? 20)
|
||||
.toArray();
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { EvalArchiveEntry } from "./archive.js";
|
||||
import {
|
||||
appendCandidatesToJsonl,
|
||||
selectPromotionCandidatesFromArchiveEntries,
|
||||
} from "./promote.js";
|
||||
|
||||
function archiveEntry(
|
||||
streamId: string,
|
||||
options: {
|
||||
intent?: "photo-question" | "status-question" | undefined;
|
||||
overall: number;
|
||||
phase?: "fresh" | "photo-pending" | undefined;
|
||||
status: "pass" | "soft-warn" | "reject";
|
||||
},
|
||||
): EvalArchiveEntry {
|
||||
return {
|
||||
streamId,
|
||||
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
request: {
|
||||
requestId: streamId,
|
||||
skill: "onboarding",
|
||||
chatroomId: "raw-room",
|
||||
userId: "raw-user",
|
||||
nickname: "홍길동",
|
||||
trigger: "user-message",
|
||||
userMessage: "전화번호는 010-1234-5678이고 인증 어디서 해요?",
|
||||
},
|
||||
response: {
|
||||
requestId: streamId,
|
||||
text: "010-1234-5678로 안내했어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: options.phase ?? "photo-pending" },
|
||||
intent: { kind: options.intent ?? "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: options.intent ?? "status-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: options.status,
|
||||
text: "응답",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: options.overall,
|
||||
accuracy: options.overall,
|
||||
concision: options.overall,
|
||||
scope: options.overall,
|
||||
overall: options.overall,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("selectPromotionCandidatesFromArchiveEntries", () => {
|
||||
it("selects low-score, disagreement, and rare-slice candidates", () => {
|
||||
const candidates = selectPromotionCandidatesFromArchiveEntries(
|
||||
[
|
||||
archiveEntry("1-0", { overall: 1, status: "pass" }),
|
||||
archiveEntry("2-0", { overall: 2.5, status: "pass" }),
|
||||
archiveEntry("3-0", {
|
||||
intent: "photo-question",
|
||||
overall: 5,
|
||||
phase: "fresh",
|
||||
status: "pass",
|
||||
}),
|
||||
],
|
||||
{
|
||||
limit: 10,
|
||||
minLowScore: 2,
|
||||
rareThreshold: 1,
|
||||
},
|
||||
);
|
||||
|
||||
expect(candidates.map((candidate) => candidate.reason)).toEqual([
|
||||
"low-score",
|
||||
"rule-llm-disagree",
|
||||
"rare-slice",
|
||||
]);
|
||||
expect(candidates[0]?.request.userId).not.toBe("raw-user");
|
||||
expect(candidates[0]?.request.userMessage).toContain("[REDACTED:phone]");
|
||||
expect(candidates[0]?.response.text).toContain("[REDACTED:phone]");
|
||||
});
|
||||
|
||||
it("prioritizes human labeled failures and carries better responses into notes", () => {
|
||||
const candidates = selectPromotionCandidatesFromArchiveEntries(
|
||||
[archiveEntry("1-0", { overall: 4, status: "pass" })],
|
||||
{
|
||||
limit: 10,
|
||||
minLowScore: 2,
|
||||
rareThreshold: 0,
|
||||
},
|
||||
[
|
||||
{
|
||||
evalStreamId: "1-0",
|
||||
verdict: "fail",
|
||||
humanScore: 1,
|
||||
rationale: "전화번호 010-1234-5678 노출",
|
||||
reviewedBy: "tester",
|
||||
reviewedAt: "2026-05-01T01:00:00.000Z",
|
||||
betterResponse: "개인정보는 이 채팅방에 남기지 말고 운영진에게 문의해 주세요.",
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
expect(candidates[0]).toMatchObject({
|
||||
reason: "labeled-fail",
|
||||
humanScore: 1,
|
||||
});
|
||||
expect(candidates[0]?.labelRationale).toContain("[REDACTED:phone]");
|
||||
expect(candidates[0]?.betterResponse).toContain("운영진");
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendCandidatesToJsonl", () => {
|
||||
it("writes candidates in eval case format and skips duplicate case ids", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "lolympus-promote-"));
|
||||
const outputPath = join(directory, "candidate.jsonl");
|
||||
const candidates = selectPromotionCandidatesFromArchiveEntries(
|
||||
[archiveEntry("1-0", { overall: 1, status: "pass" })],
|
||||
{
|
||||
limit: 10,
|
||||
minLowScore: 2,
|
||||
rareThreshold: 1,
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const first = await appendCandidatesToJsonl(candidates, outputPath);
|
||||
const second = await appendCandidatesToJsonl(candidates, outputPath);
|
||||
const raw = await readFile(outputPath, "utf8");
|
||||
|
||||
expect(first.appendedCount).toBe(1);
|
||||
expect(second.appendedCount).toBe(0);
|
||||
expect(raw.trim().split(/\r?\n/u)).toHaveLength(1);
|
||||
expect(JSON.parse(raw) as unknown).toMatchObject({
|
||||
input: {
|
||||
request: {
|
||||
skill: "onboarding",
|
||||
},
|
||||
},
|
||||
expected: {},
|
||||
});
|
||||
} finally {
|
||||
await rm(directory, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,355 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { appendFile, mkdir, readFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import {
|
||||
getEvalPromoteLowScore,
|
||||
getEvalPromotePerCategoryLimit,
|
||||
getEvalPromoteRareThreshold,
|
||||
} from "../config.js";
|
||||
import {
|
||||
anonymizeIdentifier,
|
||||
anonymizeNickname,
|
||||
anonymizeText,
|
||||
} from "./anonymize.js";
|
||||
import type { EvalCaseDefinition } from "./types.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "./archive.js";
|
||||
import { getEvalArchiveCollection } from "./archive.js";
|
||||
import { queryArchivedReviewLabels } from "./review/label-archive.js";
|
||||
import type { ReviewLabel } from "./review/queue.js";
|
||||
|
||||
const DEFAULT_CANDIDATE_PATH = "src/eval/golden/onboarding.candidate.jsonl";
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
export interface PromotionCandidate {
|
||||
betterResponse?: string | undefined;
|
||||
caseId: string;
|
||||
humanScore?: number | undefined;
|
||||
labelRationale?: string | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
derived: EvalArchiveEntry["derived"];
|
||||
llmJudge: EvalArchiveEntry["llmJudge"];
|
||||
reason: "labeled-fail" | "low-score" | "rule-llm-disagree" | "rare-slice";
|
||||
sourceStreamId: string;
|
||||
}
|
||||
|
||||
interface SelectionOptions {
|
||||
limit?: number | undefined;
|
||||
minLowScore?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
rareThreshold?: number | undefined;
|
||||
}
|
||||
|
||||
function stableShortHash(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex").slice(0, 8);
|
||||
}
|
||||
|
||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
if (entry.llmJudge.skipped) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function sliceKey(entry: EvalArchiveEntry): string {
|
||||
return `${entry.tier}:${entry.derived.intent.kind}:${entry.derived.phase.kind}`;
|
||||
}
|
||||
|
||||
function sortByLowestScore(
|
||||
left: EvalArchiveEntry,
|
||||
right: EvalArchiveEntry,
|
||||
): number {
|
||||
return (getLlmOverall(left) ?? Number.POSITIVE_INFINITY) -
|
||||
(getLlmOverall(right) ?? Number.POSITIVE_INFINITY);
|
||||
}
|
||||
|
||||
function anonymizeUnknown(value: unknown): unknown {
|
||||
if (typeof value === "string") {
|
||||
return anonymizeText(value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => anonymizeUnknown(item));
|
||||
}
|
||||
if (typeof value === "object" && value !== null) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, nested]) => [
|
||||
key,
|
||||
anonymizeUnknown(nested),
|
||||
]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function anonymizeRequest(request: AgentRequest): AgentRequest {
|
||||
const anonymized: AgentRequest = {
|
||||
requestId:
|
||||
anonymizeIdentifier("req", request.requestId) ??
|
||||
`req_${stableShortHash(request.requestId)}`,
|
||||
skill: request.skill,
|
||||
chatroomId:
|
||||
anonymizeIdentifier("room", request.chatroomId) ??
|
||||
`room_${stableShortHash(request.chatroomId)}`,
|
||||
userId:
|
||||
anonymizeIdentifier("user", request.userId) ??
|
||||
`user_${stableShortHash(request.userId)}`,
|
||||
nickname: anonymizeNickname(request.nickname) ?? "<nick:unknown>",
|
||||
trigger: request.trigger,
|
||||
};
|
||||
|
||||
if (request.userMessage !== undefined) {
|
||||
anonymized.userMessage = anonymizeText(request.userMessage);
|
||||
}
|
||||
if (request.context !== undefined) {
|
||||
anonymized.context = anonymizeUnknown(request.context) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
}
|
||||
|
||||
return anonymized;
|
||||
}
|
||||
|
||||
function anonymizeResponse(response: AgentResponse): AgentResponse {
|
||||
const anonymized: AgentResponse = {
|
||||
requestId:
|
||||
anonymizeIdentifier("req", response.requestId) ??
|
||||
`req_${stableShortHash(response.requestId)}`,
|
||||
text: anonymizeText(response.text),
|
||||
confidence: response.confidence,
|
||||
source: response.source,
|
||||
};
|
||||
|
||||
if (response.metadata !== undefined) {
|
||||
anonymized.metadata = anonymizeUnknown(response.metadata) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
}
|
||||
|
||||
return anonymized;
|
||||
}
|
||||
|
||||
function toPromotionCandidate(
|
||||
entry: EvalArchiveEntry,
|
||||
reason: PromotionCandidate["reason"],
|
||||
label?: ReviewLabel,
|
||||
): PromotionCandidate {
|
||||
const candidate: PromotionCandidate = {
|
||||
caseId:
|
||||
anonymizeIdentifier("case", entry.streamId) ??
|
||||
`case_${stableShortHash(entry.streamId)}`,
|
||||
request: anonymizeRequest(entry.request),
|
||||
response: anonymizeResponse(entry.response),
|
||||
derived: entry.derived,
|
||||
llmJudge: entry.llmJudge,
|
||||
reason,
|
||||
sourceStreamId: entry.streamId,
|
||||
};
|
||||
if (label?.betterResponse !== undefined) {
|
||||
candidate.betterResponse = anonymizeText(label.betterResponse);
|
||||
}
|
||||
if (label?.humanScore !== undefined) {
|
||||
candidate.humanScore = label.humanScore;
|
||||
}
|
||||
if (label?.rationale !== undefined) {
|
||||
candidate.labelRationale = anonymizeText(label.rationale);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function addUniqueCandidate(
|
||||
candidates: PromotionCandidate[],
|
||||
seen: Set<string>,
|
||||
entry: EvalArchiveEntry,
|
||||
reason: PromotionCandidate["reason"],
|
||||
label?: ReviewLabel,
|
||||
): void {
|
||||
if (seen.has(entry.streamId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
seen.add(entry.streamId);
|
||||
candidates.push(toPromotionCandidate(entry, reason, label));
|
||||
}
|
||||
|
||||
export function selectPromotionCandidatesFromArchiveEntries(
|
||||
entries: EvalArchiveEntry[],
|
||||
options?: SelectionOptions,
|
||||
labels: ReviewLabel[] = [],
|
||||
): PromotionCandidate[] {
|
||||
const perCategoryLimit = options?.limit ?? getEvalPromotePerCategoryLimit();
|
||||
const minLowScore = options?.minLowScore ?? getEvalPromoteLowScore();
|
||||
const rareThreshold = options?.rareThreshold ?? getEvalPromoteRareThreshold();
|
||||
const candidates: PromotionCandidate[] = [];
|
||||
const seen = new Set<string>();
|
||||
const entryByStreamId = new Map(
|
||||
entries.map((entry) => [entry.streamId, entry]),
|
||||
);
|
||||
|
||||
const labeledFail = labels
|
||||
.filter((label) => label.humanScore <= 2)
|
||||
.slice(0, perCategoryLimit);
|
||||
for (const label of labeledFail) {
|
||||
const entry = entryByStreamId.get(label.evalStreamId);
|
||||
if (entry) {
|
||||
addUniqueCandidate(candidates, seen, entry, "labeled-fail", label);
|
||||
}
|
||||
}
|
||||
|
||||
const lowScore = entries
|
||||
.filter((entry) => {
|
||||
const overall = getLlmOverall(entry);
|
||||
return overall !== null && overall <= minLowScore;
|
||||
})
|
||||
.sort(sortByLowestScore)
|
||||
.slice(0, perCategoryLimit);
|
||||
for (const entry of lowScore) {
|
||||
addUniqueCandidate(candidates, seen, entry, "low-score");
|
||||
}
|
||||
|
||||
const disagree = entries
|
||||
.filter((entry) => {
|
||||
const overall = getLlmOverall(entry);
|
||||
if (overall === null) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
(entry.validation.status === "pass" && overall <= 3) ||
|
||||
(entry.validation.status === "reject" && overall >= 4)
|
||||
);
|
||||
})
|
||||
.sort(sortByLowestScore)
|
||||
.slice(0, perCategoryLimit);
|
||||
for (const entry of disagree) {
|
||||
addUniqueCandidate(candidates, seen, entry, "rule-llm-disagree");
|
||||
}
|
||||
|
||||
const sliceCounts = new Map<string, number>();
|
||||
for (const entry of entries) {
|
||||
const key = sliceKey(entry);
|
||||
sliceCounts.set(key, (sliceCounts.get(key) ?? 0) + 1);
|
||||
}
|
||||
|
||||
const rare = entries
|
||||
.filter((entry) => (sliceCounts.get(sliceKey(entry)) ?? 0) <= rareThreshold)
|
||||
.slice(0, perCategoryLimit);
|
||||
for (const entry of rare) {
|
||||
addUniqueCandidate(candidates, seen, entry, "rare-slice");
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export async function selectPromotionCandidates(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: SelectionOptions,
|
||||
): Promise<PromotionCandidate[]> {
|
||||
const nowMs = options?.nowMs ?? Date.now();
|
||||
const [entries, labels] = await Promise.all([
|
||||
getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
evaluatedAt: {
|
||||
$gte: new Date(nowMs - DAY_MS),
|
||||
},
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(10_000)
|
||||
.toArray(),
|
||||
queryArchivedReviewLabels(mongoConnection, {
|
||||
nowMs,
|
||||
sinceMs: DAY_MS,
|
||||
}),
|
||||
]);
|
||||
|
||||
return selectPromotionCandidatesFromArchiveEntries(entries, options, labels);
|
||||
}
|
||||
|
||||
function candidateToEvalCase(candidate: PromotionCandidate): EvalCaseDefinition {
|
||||
const llmOverall = candidate.llmJudge.skipped
|
||||
? "n/a"
|
||||
: String(candidate.llmJudge.scores.overall);
|
||||
const notes = [
|
||||
`promotionReason=${candidate.reason}`,
|
||||
`sourceStreamId=${candidate.sourceStreamId}`,
|
||||
`llmOverall=${llmOverall}`,
|
||||
candidate.humanScore === undefined
|
||||
? null
|
||||
: `humanScore=${candidate.humanScore}`,
|
||||
candidate.labelRationale === undefined
|
||||
? null
|
||||
: `humanRationale=${candidate.labelRationale}`,
|
||||
`derivedIntent=${candidate.derived.intent.kind}`,
|
||||
`derivedPhase=${candidate.derived.phase.kind}`,
|
||||
`assistantResponse=${candidate.response.text}`,
|
||||
candidate.betterResponse === undefined
|
||||
? null
|
||||
: `betterResponse=${candidate.betterResponse}`,
|
||||
]
|
||||
.filter((note): note is string => note !== null)
|
||||
.join("; ");
|
||||
|
||||
return {
|
||||
id: candidate.caseId,
|
||||
lineNumber: 0,
|
||||
name: `promotion:${candidate.reason}`,
|
||||
notes,
|
||||
input: {
|
||||
request: candidate.request,
|
||||
},
|
||||
expected: {},
|
||||
};
|
||||
}
|
||||
|
||||
function extractExistingCaseIds(raw: string): Set<string> {
|
||||
const ids = new Set<string>();
|
||||
for (const line of raw.split(/\r?\n/u)) {
|
||||
if (!line.trim()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(line) as { id?: unknown };
|
||||
if (typeof parsed.id === "string") {
|
||||
ids.add(parsed.id);
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
export async function appendCandidatesToJsonl(
|
||||
candidates: PromotionCandidate[],
|
||||
outputPath = DEFAULT_CANDIDATE_PATH,
|
||||
): Promise<{ appendedCount: number; outputPath: string }> {
|
||||
const resolved = resolve(process.cwd(), outputPath);
|
||||
await mkdir(dirname(resolved), { recursive: true });
|
||||
const existingRaw = existsSync(resolved)
|
||||
? await readFile(resolved, "utf8")
|
||||
: "";
|
||||
const existingIds = extractExistingCaseIds(existingRaw);
|
||||
const lines: string[] = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (existingIds.has(candidate.caseId)) {
|
||||
continue;
|
||||
}
|
||||
lines.push(JSON.stringify(candidateToEvalCase(candidate)));
|
||||
existingIds.add(candidate.caseId);
|
||||
}
|
||||
|
||||
if (lines.length > 0) {
|
||||
await appendFile(resolved, `${lines.join("\n")}\n`, "utf8");
|
||||
}
|
||||
|
||||
return {
|
||||
appendedCount: lines.length,
|
||||
outputPath: resolved,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { EvalArchiveEntry } from "../archive.js";
|
||||
import type { ReviewLabel } from "./queue.js";
|
||||
import { computeCalibrationReportFromData } from "./calibration.js";
|
||||
|
||||
function archiveEntry(streamId: string, overall: number): EvalArchiveEntry {
|
||||
return {
|
||||
streamId,
|
||||
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
request: {
|
||||
requestId: streamId,
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "상태 알려줘",
|
||||
},
|
||||
response: {
|
||||
requestId: streamId,
|
||||
text: "사진 인증만 남았어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "사진 인증만 남았어요.",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: overall,
|
||||
accuracy: overall,
|
||||
concision: overall,
|
||||
scope: overall,
|
||||
overall,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function label(evalStreamId: string, humanScore: 1 | 2 | 3 | 4 | 5): ReviewLabel {
|
||||
return {
|
||||
evalStreamId,
|
||||
verdict: humanScore >= 4 ? "pass" : "fail",
|
||||
humanScore,
|
||||
rationale: "reviewed",
|
||||
reviewedBy: "tester",
|
||||
reviewedAt: "2026-05-01T01:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
describe("computeCalibrationReportFromData", () => {
|
||||
it("computes correlation and gaps from human/LLM score pairs", () => {
|
||||
const report = computeCalibrationReportFromData(
|
||||
[label("1-0", 1), label("2-0", 3), label("3-0", 5)],
|
||||
[
|
||||
archiveEntry("1-0", 1),
|
||||
archiveEntry("2-0", 3),
|
||||
archiveEntry("3-0", 5),
|
||||
],
|
||||
Date.parse("2026-05-01T02:00:00.000Z"),
|
||||
);
|
||||
|
||||
expect(report).toMatchObject({
|
||||
generatedAt: "2026-05-01T02:00:00.000Z",
|
||||
pairs: 3,
|
||||
pearsonOverall: 1,
|
||||
meanGapOverall: 0,
|
||||
});
|
||||
expect(report.pearsonByDim.accuracy).toBe(1);
|
||||
});
|
||||
|
||||
it("ignores labels without usable archived LLM judge data", () => {
|
||||
const skipped = archiveEntry("2-0", 3);
|
||||
skipped.llmJudge = {
|
||||
skipped: true,
|
||||
reason: "disabled",
|
||||
};
|
||||
|
||||
const report = computeCalibrationReportFromData(
|
||||
[label("1-0", 1), label("2-0", 3), label("missing", 5)],
|
||||
[archiveEntry("1-0", 2), skipped],
|
||||
Date.parse("2026-05-01T02:00:00.000Z"),
|
||||
);
|
||||
|
||||
expect(report.pairs).toBe(1);
|
||||
expect(report.meanGapOverall).toBe(-1);
|
||||
expect(report.worstDisagreements).toEqual([
|
||||
{
|
||||
evalStreamId: "1-0",
|
||||
humanScore: 1,
|
||||
llmOverall: 2,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
import {
|
||||
type EvalArchiveEntry,
|
||||
getEvalArchiveCollection,
|
||||
type MongoConnectionLike,
|
||||
} from "../archive.js";
|
||||
import { queryArchivedReviewLabels } from "./label-archive.js";
|
||||
import type { ReviewLabel } from "./queue.js";
|
||||
import { readReviewLabels } from "./queue.js";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
export interface CalibrationReport {
|
||||
generatedAt: string;
|
||||
meanGapOverall: number;
|
||||
pairs: number;
|
||||
pearsonByDim: {
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
friendliness: number;
|
||||
scope: number;
|
||||
};
|
||||
pearsonOverall: number;
|
||||
worstDisagreements: Array<{
|
||||
evalStreamId: string;
|
||||
humanScore: number;
|
||||
llmOverall: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface CalibrationPair {
|
||||
evalStreamId: string;
|
||||
humanScore: number;
|
||||
llmAccuracy: number;
|
||||
llmConcision: number;
|
||||
llmFriendliness: number;
|
||||
llmOverall: number;
|
||||
llmScope: number;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function pearson(left: number[], right: number[]): number {
|
||||
if (left.length < 2 || right.length < 2 || left.length !== right.length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const leftMean = left.reduce((sum, value) => sum + value, 0) / left.length;
|
||||
const rightMean = right.reduce((sum, value) => sum + value, 0) / right.length;
|
||||
let numerator = 0;
|
||||
let leftVariance = 0;
|
||||
let rightVariance = 0;
|
||||
|
||||
for (let index = 0; index < left.length; index++) {
|
||||
const leftDelta = (left[index] ?? 0) - leftMean;
|
||||
const rightDelta = (right[index] ?? 0) - rightMean;
|
||||
numerator += leftDelta * rightDelta;
|
||||
leftVariance += leftDelta * leftDelta;
|
||||
rightVariance += rightDelta * rightDelta;
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(leftVariance * rightVariance);
|
||||
return denominator === 0 ? 0 : round(numerator / denominator);
|
||||
}
|
||||
|
||||
function toCalibrationPairs(
|
||||
labels: ReviewLabel[],
|
||||
archives: EvalArchiveEntry[],
|
||||
): CalibrationPair[] {
|
||||
const archiveByStreamId = new Map(
|
||||
archives.map((entry) => [entry.streamId, entry]),
|
||||
);
|
||||
const pairs: CalibrationPair[] = [];
|
||||
|
||||
for (const label of labels) {
|
||||
const archive = archiveByStreamId.get(label.evalStreamId);
|
||||
if (!archive || archive.llmJudge.skipped) {
|
||||
continue;
|
||||
}
|
||||
|
||||
pairs.push({
|
||||
evalStreamId: label.evalStreamId,
|
||||
humanScore: label.humanScore,
|
||||
llmAccuracy: archive.llmJudge.scores.accuracy,
|
||||
llmConcision: archive.llmJudge.scores.concision,
|
||||
llmFriendliness: archive.llmJudge.scores.friendliness,
|
||||
llmOverall: archive.llmJudge.scores.overall,
|
||||
llmScope: archive.llmJudge.scores.scope,
|
||||
});
|
||||
}
|
||||
|
||||
return pairs;
|
||||
}
|
||||
|
||||
function mergeLabels(
|
||||
streamLabels: ReviewLabel[],
|
||||
archivedLabels: ReviewLabel[],
|
||||
): ReviewLabel[] {
|
||||
const merged: ReviewLabel[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const label of [...streamLabels, ...archivedLabels]) {
|
||||
const key = [
|
||||
label.evalStreamId,
|
||||
label.reviewedAt,
|
||||
label.reviewedBy,
|
||||
label.humanScore,
|
||||
].join(":");
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
merged.push(label);
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
||||
export function computeCalibrationReportFromData(
|
||||
labels: ReviewLabel[],
|
||||
archives: EvalArchiveEntry[],
|
||||
nowMs = Date.now(),
|
||||
): CalibrationReport {
|
||||
const pairs = toCalibrationPairs(labels, archives);
|
||||
const humanScores = pairs.map((pair) => pair.humanScore);
|
||||
const overallScores = pairs.map((pair) => pair.llmOverall);
|
||||
const meanGapOverall =
|
||||
pairs.length === 0
|
||||
? 0
|
||||
: round(
|
||||
pairs.reduce(
|
||||
(sum, pair) => sum + (pair.humanScore - pair.llmOverall),
|
||||
0,
|
||||
) / pairs.length,
|
||||
);
|
||||
|
||||
return {
|
||||
generatedAt: new Date(nowMs).toISOString(),
|
||||
meanGapOverall,
|
||||
pairs: pairs.length,
|
||||
pearsonByDim: {
|
||||
accuracy: pearson(humanScores, pairs.map((pair) => pair.llmAccuracy)),
|
||||
concision: pearson(humanScores, pairs.map((pair) => pair.llmConcision)),
|
||||
friendliness: pearson(
|
||||
humanScores,
|
||||
pairs.map((pair) => pair.llmFriendliness),
|
||||
),
|
||||
scope: pearson(humanScores, pairs.map((pair) => pair.llmScope)),
|
||||
},
|
||||
pearsonOverall: pearson(humanScores, overallScores),
|
||||
worstDisagreements: pairs
|
||||
.map((pair) => ({
|
||||
evalStreamId: pair.evalStreamId,
|
||||
humanScore: pair.humanScore,
|
||||
llmOverall: pair.llmOverall,
|
||||
}))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
Math.abs(right.humanScore - right.llmOverall) -
|
||||
Math.abs(left.humanScore - left.llmOverall),
|
||||
)
|
||||
.slice(0, 10),
|
||||
};
|
||||
}
|
||||
|
||||
export async function computeCalibrationReport(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
options?: { sinceMs?: number | undefined },
|
||||
): Promise<CalibrationReport> {
|
||||
const nowMs = Date.now();
|
||||
const [streamLabels, archivedLabels] = await Promise.all([
|
||||
readReviewLabels(garnetClient),
|
||||
queryArchivedReviewLabels(mongoConnection, {
|
||||
sinceMs: options?.sinceMs,
|
||||
nowMs,
|
||||
}),
|
||||
]);
|
||||
const labels = mergeLabels(streamLabels, archivedLabels);
|
||||
const cutoffMs =
|
||||
options?.sinceMs === undefined ? null : nowMs - options.sinceMs;
|
||||
const filteredLabels =
|
||||
cutoffMs === null
|
||||
? labels
|
||||
: labels.filter((label) => {
|
||||
const reviewedAt = Date.parse(label.reviewedAt);
|
||||
return Number.isFinite(reviewedAt) && reviewedAt >= cutoffMs;
|
||||
});
|
||||
const ids = [...new Set(filteredLabels.map((label) => label.evalStreamId))];
|
||||
const archives =
|
||||
ids.length === 0
|
||||
? []
|
||||
: await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
streamId: {
|
||||
$in: ids,
|
||||
},
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(ids.length)
|
||||
.toArray();
|
||||
|
||||
return computeCalibrationReportFromData(filteredLabels, archives);
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import {
|
||||
acknowledgeReviewItem,
|
||||
enqueueReviewCandidate,
|
||||
fetchNextReviewItem,
|
||||
recordReviewLabel,
|
||||
resetPendingReviewItems,
|
||||
type ReviewLabel,
|
||||
type ReviewQueueEntry,
|
||||
type ReviewReason,
|
||||
type ReviewConsumerGarnetClient,
|
||||
} from "./queue.js";
|
||||
|
||||
export interface ReviewCliOptions {
|
||||
consumerName?: string | undefined;
|
||||
exportLabeled?: string | undefined;
|
||||
limit?: number | undefined;
|
||||
reason?: ReviewReason | undefined;
|
||||
resetPending?: boolean | undefined;
|
||||
}
|
||||
|
||||
function getReviewerId(): string {
|
||||
const configured = process.env["REVIEWER_ID"]?.trim();
|
||||
if (configured) {
|
||||
return configured;
|
||||
}
|
||||
|
||||
try {
|
||||
return execFileSync("git", ["config", "user.email"], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim() || "unknown";
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function formatLlmJudge(entry: ReviewQueueEntry): string {
|
||||
if (entry.llmJudge.skipped) {
|
||||
return `skipped=${entry.llmJudge.reason}`;
|
||||
}
|
||||
|
||||
return [
|
||||
`overall=${entry.llmJudge.scores.overall}`,
|
||||
`friendliness=${entry.llmJudge.scores.friendliness}`,
|
||||
`accuracy=${entry.llmJudge.scores.accuracy}`,
|
||||
`concision=${entry.llmJudge.scores.concision}`,
|
||||
`scope=${entry.llmJudge.scores.scope}`,
|
||||
`rationale=${entry.llmJudge.rationale}`,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function formatProcess(entry: ReviewQueueEntry): string {
|
||||
const steps = entry.process?.steps;
|
||||
if (!steps) {
|
||||
return "n/a";
|
||||
}
|
||||
|
||||
return [
|
||||
steps.classify ? "classify" : null,
|
||||
steps.derive ? "derive" : null,
|
||||
steps.plan ? "plan" : null,
|
||||
...(steps.toolCalls?.map((call) => `tool:${call.output.name}`) ?? []),
|
||||
steps.respond ? "respond" : null,
|
||||
steps.critic ? "critic" : null,
|
||||
steps.validate ? "validate" : null,
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join(" -> ");
|
||||
}
|
||||
|
||||
function printReviewItem(
|
||||
entry: ReviewQueueEntry,
|
||||
index: number,
|
||||
limit: number,
|
||||
): void {
|
||||
console.info("");
|
||||
console.info(`[${index}/${limit}] reason: ${entry.reason}`);
|
||||
console.info(
|
||||
`chatroom: ${entry.request.chatroomId} / user: ${entry.request.userId} / trigger: ${entry.request.trigger}`,
|
||||
);
|
||||
console.info(
|
||||
`intent: ${entry.classified.intent.kind} (confidence ${entry.classified.confidence}, ${entry.classified.source})`,
|
||||
);
|
||||
console.info(
|
||||
`tier: ${entry.derived.tier} / phase: ${entry.derived.phase.kind} / responseMode: ${entry.derived.responseMode}`,
|
||||
);
|
||||
console.info("");
|
||||
console.info("[user message]");
|
||||
console.info(entry.request.userMessage ?? "(no user message)");
|
||||
console.info("");
|
||||
console.info("[assistant response]");
|
||||
console.info(entry.response.text);
|
||||
console.info("");
|
||||
console.info(`[llm judge] ${formatLlmJudge(entry)}`);
|
||||
console.info(`[validator] status=${entry.validation.status} reasons=${entry.validation.reasons.join(",") || "-"}`);
|
||||
console.info(`[process] ${formatProcess(entry)}`);
|
||||
console.info("");
|
||||
}
|
||||
|
||||
async function askVerdict(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<"fail" | "pass" | "quit" | "skip"> {
|
||||
while (true) {
|
||||
const answer = (await rl.question("verdict (p=pass / f=fail / s=skip / q=quit) > "))
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (answer === "p") return "pass";
|
||||
if (answer === "f") return "fail";
|
||||
if (answer === "s") return "skip";
|
||||
if (answer === "q") return "quit";
|
||||
}
|
||||
}
|
||||
|
||||
async function askHumanScore(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<1 | 2 | 3 | 4 | 5> {
|
||||
while (true) {
|
||||
const answer = Number((await rl.question("humanScore (1-5) > ")).trim());
|
||||
if (answer === 1 || answer === 2 || answer === 3 || answer === 4 || answer === 5) {
|
||||
return answer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function askNonEmpty(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
prompt: string,
|
||||
): Promise<string> {
|
||||
while (true) {
|
||||
const answer = (await rl.question(prompt)).trim();
|
||||
if (answer) {
|
||||
return answer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function exportLabels(
|
||||
labels: ReviewLabel[],
|
||||
outputPath: string,
|
||||
): Promise<string> {
|
||||
const resolved = resolve(process.cwd(), outputPath);
|
||||
await mkdir(dirname(resolved), { recursive: true });
|
||||
await writeFile(
|
||||
resolved,
|
||||
labels.map((label) => JSON.stringify(label)).join("\n") + "\n",
|
||||
"utf8",
|
||||
);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function isReviewReason(value: string | undefined): value is ReviewReason {
|
||||
return (
|
||||
value === "low-score" ||
|
||||
value === "rare-slice" ||
|
||||
value === "rule-llm-disagree" ||
|
||||
value === "validator-soft-warn"
|
||||
);
|
||||
}
|
||||
|
||||
export function parseReviewReason(value: string | undefined): ReviewReason | undefined {
|
||||
return isReviewReason(value) ? value : undefined;
|
||||
}
|
||||
|
||||
export async function runReviewCli(
|
||||
garnetClient: Pick<
|
||||
GarnetClient,
|
||||
| "acknowledgeStream"
|
||||
| "appendStream"
|
||||
| "claimPendingStream"
|
||||
| "createConsumerGroup"
|
||||
| "getPendingStreamRange"
|
||||
| "readGroup"
|
||||
>,
|
||||
options?: ReviewCliOptions,
|
||||
): Promise<{ labeled: number; skipped: number }> {
|
||||
const rl = createInterface({ input, output });
|
||||
const limit = options?.limit ?? 20;
|
||||
const consumerName = options?.consumerName ?? `review-cli-${process.pid}`;
|
||||
const reviewedBy = getReviewerId();
|
||||
const labels: ReviewLabel[] = [];
|
||||
let skipped = 0;
|
||||
let mismatchAttempts = 0;
|
||||
|
||||
try {
|
||||
if (options?.resetPending) {
|
||||
const result = await resetPendingReviewItems(garnetClient, {
|
||||
consumerName,
|
||||
nowMs: Date.now(),
|
||||
});
|
||||
console.info(`[eval] resetPending=${result.resetCount}`);
|
||||
}
|
||||
|
||||
for (let index = 1; index <= limit; index++) {
|
||||
const item = await fetchNextReviewItem(garnetClient as ReviewConsumerGarnetClient, {
|
||||
consumerName,
|
||||
blockMs: 5_000,
|
||||
});
|
||||
if (!item) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (options?.reason && item.entry.reason !== options.reason) {
|
||||
await enqueueReviewCandidate(garnetClient, {
|
||||
...item.entry,
|
||||
enqueuedAt: new Date().toISOString(),
|
||||
});
|
||||
await acknowledgeReviewItem(garnetClient, item.streamId);
|
||||
mismatchAttempts += 1;
|
||||
if (mismatchAttempts >= limit * 5) {
|
||||
break;
|
||||
}
|
||||
index -= 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
printReviewItem(item.entry, index, limit);
|
||||
const verdict = await askVerdict(rl);
|
||||
if (verdict === "quit") {
|
||||
break;
|
||||
}
|
||||
if (verdict === "skip") {
|
||||
await acknowledgeReviewItem(garnetClient, item.streamId);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const humanScore = await askHumanScore(rl);
|
||||
const rationale = await askNonEmpty(rl, "rationale > ");
|
||||
const betterResponse = (await rl.question("better (optional) > ")).trim();
|
||||
const label: ReviewLabel = {
|
||||
evalStreamId: item.entry.evalStreamId,
|
||||
verdict,
|
||||
humanScore,
|
||||
rationale,
|
||||
reviewedBy,
|
||||
reviewedAt: new Date().toISOString(),
|
||||
};
|
||||
if (betterResponse) {
|
||||
label.betterResponse = betterResponse;
|
||||
}
|
||||
|
||||
await recordReviewLabel(garnetClient, {
|
||||
consumerName,
|
||||
label,
|
||||
streamId: item.streamId,
|
||||
});
|
||||
labels.push(label);
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
|
||||
if (options?.exportLabeled && labels.length > 0) {
|
||||
const outputPath = await exportLabels(labels, options.exportLabeled);
|
||||
console.info(`[eval] exported labeled=${labels.length} output=${outputPath}`);
|
||||
}
|
||||
|
||||
return {
|
||||
labeled: labels.length,
|
||||
skipped,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
archiveReviewLabels,
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION,
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
queryArchivedReviewLabels,
|
||||
type ReviewLabelArchiveEntry,
|
||||
} from "./label-archive.js";
|
||||
import { EVAL_REVIEW_LABELED_STREAM_KEY, type ReviewLabel } from "./queue.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly strings = new Map<string, string>();
|
||||
|
||||
constructor(private readonly entries: StreamEntry[]) {}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
_end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(EVAL_REVIEW_LABELED_STREAM_KEY);
|
||||
const filtered = start.startsWith("(")
|
||||
? this.entries.filter((entry) => entry.id > start.slice(1))
|
||||
: this.entries;
|
||||
return Promise.resolve(filtered.slice(0, options?.count));
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryCursor {
|
||||
constructor(private readonly entries: ReviewLabelArchiveEntry[]) {}
|
||||
|
||||
sort(): MemoryCursor {
|
||||
this.entries.sort(
|
||||
(left, right) => right.reviewedAt.getTime() - left.reviewedAt.getTime(),
|
||||
);
|
||||
return this;
|
||||
}
|
||||
|
||||
limit(value: number): MemoryCursor {
|
||||
this.entries.splice(value);
|
||||
return this;
|
||||
}
|
||||
|
||||
toArray(): Promise<ReviewLabelArchiveEntry[]> {
|
||||
return Promise.resolve(this.entries);
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryCollection {
|
||||
readonly docs = new Map<string, ReviewLabelArchiveEntry>();
|
||||
readonly indexes: Array<Record<string, unknown>> = [];
|
||||
|
||||
bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> {
|
||||
let upsertedCount = 0;
|
||||
for (const operation of operations) {
|
||||
const updateOne = (operation as {
|
||||
updateOne: {
|
||||
filter: { labelStreamId: string };
|
||||
update: { $setOnInsert: ReviewLabelArchiveEntry };
|
||||
};
|
||||
}).updateOne;
|
||||
const streamId = updateOne.filter.labelStreamId;
|
||||
if (!this.docs.has(streamId)) {
|
||||
this.docs.set(streamId, updateOne.update.$setOnInsert);
|
||||
upsertedCount += 1;
|
||||
}
|
||||
}
|
||||
return Promise.resolve({ upsertedCount });
|
||||
}
|
||||
|
||||
createIndex(keys: Record<string, unknown>): Promise<string> {
|
||||
this.indexes.push(keys);
|
||||
return Promise.resolve("idx");
|
||||
}
|
||||
|
||||
find(filter: Record<string, unknown>): MemoryCursor {
|
||||
const reviewedAt = filter["reviewedAt"] as
|
||||
| { $gte?: Date | undefined }
|
||||
| undefined;
|
||||
const entries = [...this.docs.values()].filter((entry) => {
|
||||
if (!reviewedAt?.$gte) {
|
||||
return true;
|
||||
}
|
||||
return entry.reviewedAt >= reviewedAt.$gte;
|
||||
});
|
||||
return new MemoryCursor(entries);
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryConnection {
|
||||
readonly collectionInstance = new MemoryCollection();
|
||||
|
||||
collection(name: string): MemoryCollection {
|
||||
expect(name).toBe(EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION);
|
||||
return this.collectionInstance;
|
||||
}
|
||||
}
|
||||
|
||||
function streamEntry(id: string, label: ReviewLabel): StreamEntry {
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
type: "eval-review-label",
|
||||
evalStreamId: label.evalStreamId,
|
||||
reviewedAt: label.reviewedAt,
|
||||
reviewedBy: label.reviewedBy,
|
||||
payload: JSON.stringify(label),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("archiveReviewLabels", () => {
|
||||
it("archives label stream entries idempotently and queries them back", async () => {
|
||||
const label: ReviewLabel = {
|
||||
evalStreamId: "1-0",
|
||||
verdict: "fail",
|
||||
humanScore: 2,
|
||||
rationale: "wrong channel",
|
||||
reviewedBy: "tester",
|
||||
reviewedAt: "2026-05-01T00:01:00.000Z",
|
||||
};
|
||||
const garnet = new MemoryGarnet([streamEntry("1-0", label)]);
|
||||
const connection = new MemoryConnection();
|
||||
|
||||
const first = await archiveReviewLabels(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:02:00.000Z"),
|
||||
});
|
||||
const second = await archiveReviewLabels(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:02:00.000Z"),
|
||||
});
|
||||
const labels = await queryArchivedReviewLabels(connection);
|
||||
|
||||
expect(first).toEqual({ archivedCount: 1, lastArchivedId: "1-0" });
|
||||
expect(second).toEqual({ archivedCount: 0, lastArchivedId: "1-0" });
|
||||
expect(garnet.strings.get(EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY)).toBe("1-0");
|
||||
expect(connection.collectionInstance.indexes).toContainEqual({
|
||||
labelStreamId: 1,
|
||||
});
|
||||
expect(labels).toEqual([label]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import { getEvalArchiveBatchSize } from "../../config.js";
|
||||
import type { MongoConnectionLike } from "../archive.js";
|
||||
import {
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
parseReviewLabelEntry,
|
||||
type ReviewLabel,
|
||||
} from "./queue.js";
|
||||
|
||||
export const EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY =
|
||||
"lolympus-agent:eval:review:label-archive:cursor";
|
||||
export const EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION = "eval_review_labels";
|
||||
|
||||
export interface ReviewLabelArchiveEntry
|
||||
extends Omit<ReviewLabel, "reviewedAt"> {
|
||||
labelStreamId: string;
|
||||
reviewedAt: Date;
|
||||
}
|
||||
|
||||
interface BulkWriteResultLike {
|
||||
upsertedCount?: number | undefined;
|
||||
}
|
||||
|
||||
interface FindCursorLike<T> {
|
||||
limit(value: number): FindCursorLike<T>;
|
||||
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
|
||||
toArray(): Promise<T[]>;
|
||||
}
|
||||
|
||||
interface ReviewLabelArchiveCollectionLike {
|
||||
bulkWrite(
|
||||
operations: unknown[],
|
||||
options?: { ordered?: boolean | undefined },
|
||||
): Promise<BulkWriteResultLike>;
|
||||
createIndex(
|
||||
keys: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
): Promise<unknown>;
|
||||
find(filter: Record<string, unknown>): FindCursorLike<ReviewLabelArchiveEntry>;
|
||||
}
|
||||
|
||||
type LabelArchiveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"readStreamRange" | "setString"
|
||||
>;
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function parseReviewedAt(value: string, nowMs: number): Date {
|
||||
const parsed = new Date(value);
|
||||
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
|
||||
}
|
||||
|
||||
function toArchiveEntry(
|
||||
streamEntry: StreamEntry,
|
||||
nowMs: number,
|
||||
): ReviewLabelArchiveEntry {
|
||||
const label = parseReviewLabelEntry(streamEntry);
|
||||
return {
|
||||
...label,
|
||||
labelStreamId: streamEntry.id,
|
||||
reviewedAt: parseReviewedAt(label.reviewedAt, nowMs),
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureReviewLabelArchiveIndexes(
|
||||
collection: ReviewLabelArchiveCollectionLike,
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ labelStreamId: 1 }, { unique: true }),
|
||||
collection.createIndex({ evalStreamId: 1 }),
|
||||
collection.createIndex({ reviewedAt: -1 }),
|
||||
]);
|
||||
}
|
||||
|
||||
export function getReviewLabelArchiveCollection(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
): ReviewLabelArchiveCollectionLike {
|
||||
return mongoConnection.collection(
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION,
|
||||
) as ReviewLabelArchiveCollectionLike;
|
||||
}
|
||||
|
||||
export async function archiveReviewLabels(
|
||||
garnetClient: LabelArchiveGarnetClient,
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options: {
|
||||
batchSize?: number | undefined;
|
||||
nowMs: number;
|
||||
sinceId: string;
|
||||
},
|
||||
): Promise<{ archivedCount: number; lastArchivedId: string }> {
|
||||
const collection = getReviewLabelArchiveCollection(mongoConnection);
|
||||
await ensureReviewLabelArchiveIndexes(collection);
|
||||
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
{
|
||||
count: options.batchSize ?? getEvalArchiveBatchSize(),
|
||||
},
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
archivedCount: 0,
|
||||
lastArchivedId: options.sinceId,
|
||||
};
|
||||
}
|
||||
|
||||
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
|
||||
const archiveEntries = entries.flatMap((entry) => {
|
||||
try {
|
||||
return [toArchiveEntry(entry, options.nowMs)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
let archivedCount = 0;
|
||||
if (archiveEntries.length > 0) {
|
||||
const result = await collection.bulkWrite(
|
||||
archiveEntries.map((entry) => ({
|
||||
updateOne: {
|
||||
filter: { labelStreamId: entry.labelStreamId },
|
||||
update: { $setOnInsert: entry },
|
||||
upsert: true,
|
||||
},
|
||||
})),
|
||||
{ ordered: false },
|
||||
);
|
||||
archivedCount = result.upsertedCount ?? 0;
|
||||
}
|
||||
|
||||
await garnetClient.setString(
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
lastArchivedId,
|
||||
);
|
||||
|
||||
return {
|
||||
archivedCount,
|
||||
lastArchivedId,
|
||||
};
|
||||
}
|
||||
|
||||
export async function queryArchivedReviewLabels(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
sinceMs?: number | undefined;
|
||||
},
|
||||
): Promise<ReviewLabel[]> {
|
||||
const filter: Record<string, unknown> = {};
|
||||
if (options?.sinceMs !== undefined) {
|
||||
filter["reviewedAt"] = {
|
||||
$gte: new Date((options.nowMs ?? Date.now()) - options.sinceMs),
|
||||
};
|
||||
}
|
||||
|
||||
const entries = await getReviewLabelArchiveCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ reviewedAt: -1 })
|
||||
.limit(options?.limit ?? 10_000)
|
||||
.toArray();
|
||||
|
||||
return entries.map((entry) => {
|
||||
const label: ReviewLabel = {
|
||||
evalStreamId: entry.evalStreamId,
|
||||
verdict: entry.verdict,
|
||||
humanScore: entry.humanScore,
|
||||
rationale: entry.rationale,
|
||||
reviewedBy: entry.reviewedBy,
|
||||
reviewedAt: entry.reviewedAt.toISOString(),
|
||||
};
|
||||
if (entry.betterResponse !== undefined) {
|
||||
label.betterResponse = entry.betterResponse;
|
||||
}
|
||||
return label;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry, StreamReadResult } from "@common/garnet";
|
||||
import {
|
||||
EVAL_REVIEW_CURSOR_KEY,
|
||||
EVAL_REVIEW_GROUP,
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
enqueueReviewCandidate,
|
||||
fetchNextReviewItem,
|
||||
recordReviewLabel,
|
||||
resetPendingReviewItems,
|
||||
routeReviewCandidates,
|
||||
type ReviewQueueEntry,
|
||||
} from "./queue.js";
|
||||
import { EVAL_RESULTS_STREAM_KEY, getSliceAggregateKey } from "../aggregator.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly acknowledged: Array<{ group: string; id: string; key: string }> = [];
|
||||
readonly dedupe = new Set<string>();
|
||||
readonly groups = new Set<string>();
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly streams = new Map<string, StreamEntry[]>();
|
||||
readonly strings = new Map<string, string>();
|
||||
|
||||
acknowledgeStream(key: string, group: string, ids: string | string[]): Promise<number> {
|
||||
for (const id of Array.isArray(ids) ? ids : [ids]) {
|
||||
this.acknowledged.push({ key, group, id });
|
||||
}
|
||||
return Promise.resolve(Array.isArray(ids) ? ids.length : 1);
|
||||
}
|
||||
|
||||
appendStream(key: string, fields: Record<string, string>): Promise<string> {
|
||||
const entries = this.streams.get(key) ?? [];
|
||||
const id = `${entries.length + 1}-0`;
|
||||
entries.push({ id, message: fields });
|
||||
this.streams.set(key, entries);
|
||||
return Promise.resolve(id);
|
||||
}
|
||||
|
||||
createConsumerGroup(key: string, group: string): Promise<void> {
|
||||
this.groups.add(`${key}:${group}`);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
claimPendingStream(
|
||||
key: string,
|
||||
_group: string,
|
||||
_consumer: string,
|
||||
_minIdleTimeMs: number,
|
||||
ids: string | string[],
|
||||
): Promise<StreamEntry[]> {
|
||||
const idSet = new Set(Array.isArray(ids) ? ids : [ids]);
|
||||
return Promise.resolve(
|
||||
(this.streams.get(key) ?? []).filter((entry) => idSet.has(entry.id)),
|
||||
);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
getPendingStreamRange(
|
||||
key: string,
|
||||
group: string,
|
||||
count: number,
|
||||
): Promise<
|
||||
Array<{
|
||||
consumer: string;
|
||||
deliveriesCounter: number;
|
||||
id: string;
|
||||
millisecondsSinceLastDelivery: number;
|
||||
}>
|
||||
> {
|
||||
const acked = new Set(
|
||||
this.acknowledged
|
||||
.filter((entry) => entry.key === key && entry.group === group)
|
||||
.map((entry) => entry.id),
|
||||
);
|
||||
return Promise.resolve(
|
||||
(this.streams.get(key) ?? [])
|
||||
.filter((entry) => !acked.has(entry.id))
|
||||
.slice(0, count)
|
||||
.map((entry) => ({
|
||||
consumer: "old-reviewer",
|
||||
deliveriesCounter: 1,
|
||||
id: entry.id,
|
||||
millisecondsSinceLastDelivery: 3_600_000,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
readGroup(
|
||||
key: string,
|
||||
group: string,
|
||||
_consumer: string,
|
||||
): Promise<StreamReadResult[] | null> {
|
||||
const entries = this.streams.get(key) ?? [];
|
||||
const acked = new Set(
|
||||
this.acknowledged
|
||||
.filter((entry) => entry.key === key && entry.group === group)
|
||||
.map((entry) => entry.id),
|
||||
);
|
||||
const next = entries.find((entry) => !acked.has(entry.id));
|
||||
return Promise.resolve(
|
||||
next
|
||||
? [
|
||||
{
|
||||
name: key,
|
||||
messages: [next],
|
||||
},
|
||||
]
|
||||
: null,
|
||||
);
|
||||
}
|
||||
|
||||
readStreamRange(key: string, start: string): Promise<StreamEntry[]> {
|
||||
const entries = this.streams.get(key) ?? [];
|
||||
if (!start.startsWith("(")) {
|
||||
return Promise.resolve(entries);
|
||||
}
|
||||
const sinceId = start.slice(1);
|
||||
return Promise.resolve(entries.filter((entry) => entry.id > sinceId));
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
setStringIfAbsent(key: string): Promise<boolean> {
|
||||
if (this.dedupe.has(key)) {
|
||||
return Promise.resolve(false);
|
||||
}
|
||||
this.dedupe.add(key);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
}
|
||||
|
||||
function reviewEntry(): ReviewQueueEntry {
|
||||
return {
|
||||
evalStreamId: "1-0",
|
||||
reason: "low-score",
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "사진 어디로 보내요?",
|
||||
},
|
||||
response: {
|
||||
requestId: "req-1",
|
||||
text: "디스코드에 보내주세요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "photo-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "디스코드에 보내주세요.",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "cliproxy",
|
||||
scores: {
|
||||
friendliness: 4,
|
||||
accuracy: 1,
|
||||
concision: 4,
|
||||
scope: 2,
|
||||
overall: 2.75,
|
||||
},
|
||||
rationale: "wrong channel",
|
||||
raw: "{}",
|
||||
},
|
||||
enqueuedAt: "2026-05-01T00:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
function evalResultEntry(id: string, overall: number): StreamEntry {
|
||||
const entry = reviewEntry();
|
||||
const llmJudge = entry.llmJudge.skipped
|
||||
? entry.llmJudge
|
||||
: {
|
||||
...entry.llmJudge,
|
||||
scores: {
|
||||
...entry.llmJudge.scores,
|
||||
overall,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
input: JSON.stringify({ request: entry.request }),
|
||||
output: JSON.stringify(entry.response),
|
||||
derived: JSON.stringify(entry.derived),
|
||||
classified: JSON.stringify(entry.classified),
|
||||
validation: JSON.stringify(entry.validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("review queue", () => {
|
||||
it("enqueues, fetches, labels, and acknowledges review items", async () => {
|
||||
const garnet = new MemoryGarnet();
|
||||
const streamId = await enqueueReviewCandidate(garnet, reviewEntry());
|
||||
|
||||
const item = await fetchNextReviewItem(garnet, {
|
||||
consumerName: "reviewer-1",
|
||||
blockMs: 1,
|
||||
});
|
||||
expect(streamId).toBe("1-0");
|
||||
expect(item?.entry.reason).toBe("low-score");
|
||||
expect(garnet.groups.has(`${EVAL_REVIEW_QUEUE_STREAM_KEY}:${EVAL_REVIEW_GROUP}`)).toBe(true);
|
||||
|
||||
await recordReviewLabel(garnet, {
|
||||
consumerName: "reviewer-1",
|
||||
streamId,
|
||||
label: {
|
||||
evalStreamId: "1-0",
|
||||
verdict: "fail",
|
||||
humanScore: 1,
|
||||
rationale: "wrong channel",
|
||||
reviewedBy: "tester",
|
||||
reviewedAt: "2026-05-01T00:01:00.000Z",
|
||||
},
|
||||
});
|
||||
|
||||
expect(garnet.streams.get(EVAL_REVIEW_LABELED_STREAM_KEY)).toHaveLength(1);
|
||||
expect(garnet.acknowledged).toContainEqual({
|
||||
key: EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
group: EVAL_REVIEW_GROUP,
|
||||
id: streamId,
|
||||
});
|
||||
});
|
||||
|
||||
it("routes low-score eval results once with dedupe and cursor tracking", async () => {
|
||||
const garnet = new MemoryGarnet();
|
||||
const sliceKey = getSliceAggregateKey({
|
||||
tier: 0,
|
||||
intent: "photo-question",
|
||||
phase: "photo-pending",
|
||||
});
|
||||
garnet.hashes.set(sliceKey, { count: "10" });
|
||||
garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [
|
||||
evalResultEntry("1-0", 2),
|
||||
evalResultEntry("2-0", 4),
|
||||
]);
|
||||
|
||||
const first = await routeReviewCandidates(garnet, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
const second = await routeReviewCandidates(garnet, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(first.enqueuedCount).toBe(1);
|
||||
expect(second.enqueuedCount).toBe(0);
|
||||
expect(garnet.strings.get(EVAL_REVIEW_CURSOR_KEY)).toBe("2-0");
|
||||
expect(garnet.streams.get(EVAL_REVIEW_QUEUE_STREAM_KEY)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("resets stale pending review items by re-enqueueing and acknowledging originals", async () => {
|
||||
const garnet = new MemoryGarnet();
|
||||
const streamId = await enqueueReviewCandidate(garnet, reviewEntry());
|
||||
|
||||
const result = await resetPendingReviewItems(garnet, {
|
||||
consumerName: "reviewer-1",
|
||||
nowMs: Date.parse("2026-05-01T00:02:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result.resetCount).toBe(1);
|
||||
expect(garnet.acknowledged).toContainEqual({
|
||||
key: EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
group: EVAL_REVIEW_GROUP,
|
||||
id: streamId,
|
||||
});
|
||||
expect(garnet.streams.get(EVAL_REVIEW_QUEUE_STREAM_KEY)).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,506 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type { ValidationResult } from "@skills/onboarding/runtime/validator";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
getEvalReviewDedupeTtlSec,
|
||||
getEvalReviewLabelMaxLen,
|
||||
getEvalReviewPerCategoryLimit,
|
||||
getEvalReviewQueueMaxLen,
|
||||
} from "../../config.js";
|
||||
import {
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
getSliceAggregateKey,
|
||||
type SliceKey,
|
||||
} from "../aggregator.js";
|
||||
import type { LlmJudgeResult, ProcessJudgeResult } from "../types.js";
|
||||
|
||||
export const EVAL_REVIEW_QUEUE_STREAM_KEY =
|
||||
"lolympus-agent:eval:review:queue";
|
||||
export const EVAL_REVIEW_LABELED_STREAM_KEY =
|
||||
"lolympus-agent:eval:review:labeled";
|
||||
export const EVAL_REVIEW_CURSOR_KEY = "lolympus-agent:eval:review:cursor";
|
||||
export const EVAL_REVIEW_GROUP = "reviewers";
|
||||
|
||||
export type ReviewReason =
|
||||
| "low-score"
|
||||
| "rare-slice"
|
||||
| "rule-llm-disagree"
|
||||
| "validator-soft-warn";
|
||||
|
||||
export interface ReviewQueueEntry {
|
||||
evalStreamId: string;
|
||||
reason: ReviewReason;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
derived: DerivedState;
|
||||
classified: ClassifiedIntent;
|
||||
validation: ValidationResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
enqueuedAt: string;
|
||||
}
|
||||
|
||||
export interface ReviewLabel {
|
||||
betterResponse?: string | undefined;
|
||||
evalStreamId: string;
|
||||
humanScore: 1 | 2 | 3 | 4 | 5;
|
||||
rationale: string;
|
||||
reviewedAt: string;
|
||||
reviewedBy: string;
|
||||
verdict: "fail" | "pass";
|
||||
}
|
||||
|
||||
type ReviewRouteGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
| "acknowledgeStream"
|
||||
| "appendStream"
|
||||
| "createConsumerGroup"
|
||||
| "getHash"
|
||||
| "readGroup"
|
||||
| "readStreamRange"
|
||||
| "setString"
|
||||
| "setStringIfAbsent"
|
||||
>;
|
||||
|
||||
export type ReviewConsumerGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"createConsumerGroup" | "readGroup"
|
||||
>;
|
||||
|
||||
type ReviewPendingGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
| "acknowledgeStream"
|
||||
| "appendStream"
|
||||
| "claimPendingStream"
|
||||
| "createConsumerGroup"
|
||||
| "getPendingStreamRange"
|
||||
>;
|
||||
|
||||
interface ParsedEvalResult {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
evalStreamId: string;
|
||||
llmJudge: LlmJudgeResult;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
validation: ValidationResult;
|
||||
}
|
||||
|
||||
function parseJsonField<T>(
|
||||
value: string | undefined,
|
||||
fieldName: string,
|
||||
): T {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`Missing ${fieldName} field`);
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseFiniteNumber(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function getLlmOverall(llmJudge: LlmJudgeResult): number | null {
|
||||
return llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function sliceFromDerived(derived: DerivedState): SliceKey {
|
||||
return {
|
||||
tier: derived.tier,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
};
|
||||
}
|
||||
|
||||
function parseEvalResult(entry: StreamEntry): ParsedEvalResult | null {
|
||||
try {
|
||||
const input = parseJsonField<{ request: AgentRequest }>(
|
||||
entry.message["input"],
|
||||
"input",
|
||||
);
|
||||
const parsed: ParsedEvalResult = {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
entry.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
derived: parseJsonField<DerivedState>(
|
||||
entry.message["derived"],
|
||||
"derived",
|
||||
),
|
||||
evalStreamId: entry.id,
|
||||
llmJudge: parseJsonField<LlmJudgeResult>(
|
||||
entry.message["llmJudge"],
|
||||
"llmJudge",
|
||||
),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(
|
||||
entry.message["output"],
|
||||
"output",
|
||||
),
|
||||
validation: parseJsonField<ValidationResult>(
|
||||
entry.message["validation"],
|
||||
"validation",
|
||||
),
|
||||
};
|
||||
const processJudge = parseOptionalJsonField<ProcessJudgeResult>(
|
||||
entry.message["processJudge"],
|
||||
);
|
||||
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
|
||||
entry.message["process"],
|
||||
);
|
||||
if (processJudge) {
|
||||
parsed.processJudge = processJudge;
|
||||
}
|
||||
if (process) {
|
||||
parsed.process = process;
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function toQueueEntry(
|
||||
parsed: ParsedEvalResult,
|
||||
reason: ReviewReason,
|
||||
nowMs: number,
|
||||
): ReviewQueueEntry {
|
||||
const entry: ReviewQueueEntry = {
|
||||
evalStreamId: parsed.evalStreamId,
|
||||
reason,
|
||||
request: parsed.request,
|
||||
response: parsed.response,
|
||||
derived: parsed.derived,
|
||||
classified: parsed.classified,
|
||||
validation: parsed.validation,
|
||||
llmJudge: parsed.llmJudge,
|
||||
enqueuedAt: new Date(nowMs).toISOString(),
|
||||
};
|
||||
if (parsed.processJudge) {
|
||||
entry.processJudge = parsed.processJudge;
|
||||
}
|
||||
if (parsed.process) {
|
||||
entry.process = parsed.process;
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
function getCandidateReasons(parsed: ParsedEvalResult): ReviewReason[] {
|
||||
const reasons: ReviewReason[] = [];
|
||||
const overall = getLlmOverall(parsed.llmJudge);
|
||||
|
||||
if (overall !== null && overall <= 2) {
|
||||
reasons.push("low-score");
|
||||
}
|
||||
if (
|
||||
overall !== null &&
|
||||
((parsed.validation.status === "pass" && overall <= 3) ||
|
||||
(parsed.validation.status === "reject" && overall >= 4))
|
||||
) {
|
||||
reasons.push("rule-llm-disagree");
|
||||
}
|
||||
if (parsed.validation.status === "soft-warn") {
|
||||
reasons.push("validator-soft-warn");
|
||||
}
|
||||
|
||||
return reasons;
|
||||
}
|
||||
|
||||
function reasonLimitKey(reason: ReviewReason): ReviewReason {
|
||||
return reason;
|
||||
}
|
||||
|
||||
async function isRareSlice(
|
||||
garnetClient: Pick<GarnetClient, "getHash">,
|
||||
parsed: ParsedEvalResult,
|
||||
): Promise<boolean> {
|
||||
const aggregate = await garnetClient.getHash(
|
||||
getSliceAggregateKey(sliceFromDerived(parsed.derived)),
|
||||
);
|
||||
const count = parseFiniteNumber(aggregate["count"]) ?? 0;
|
||||
return count > 0 && count <= 5;
|
||||
}
|
||||
|
||||
async function ensureReviewConsumerGroup(
|
||||
garnetClient: Pick<GarnetClient, "createConsumerGroup">,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await garnetClient.createConsumerGroup(
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
EVAL_REVIEW_GROUP,
|
||||
"0",
|
||||
);
|
||||
} catch (error) {
|
||||
if (!String(error).includes("BUSYGROUP")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function enqueueReviewCandidate(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
entry: ReviewQueueEntry,
|
||||
): Promise<string> {
|
||||
return garnetClient.appendStream(
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
{
|
||||
type: "eval-review",
|
||||
evalStreamId: entry.evalStreamId,
|
||||
reason: entry.reason,
|
||||
payload: JSON.stringify(entry),
|
||||
enqueuedAt: entry.enqueuedAt,
|
||||
},
|
||||
{
|
||||
maxLen: getEvalReviewQueueMaxLen(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function parseReviewQueueEntry(
|
||||
streamEntry: StreamEntry,
|
||||
): ReviewQueueEntry {
|
||||
return parseJsonField<ReviewQueueEntry>(
|
||||
streamEntry.message["payload"],
|
||||
"payload",
|
||||
);
|
||||
}
|
||||
|
||||
export function parseReviewLabelEntry(streamEntry: StreamEntry): ReviewLabel {
|
||||
return parseJsonField<ReviewLabel>(streamEntry.message["payload"], "payload");
|
||||
}
|
||||
|
||||
export async function fetchNextReviewItem(
|
||||
garnetClient: ReviewConsumerGarnetClient,
|
||||
options: {
|
||||
blockMs?: number | undefined;
|
||||
consumerName: string;
|
||||
},
|
||||
): Promise<{ entry: ReviewQueueEntry; streamId: string } | null> {
|
||||
await ensureReviewConsumerGroup(garnetClient);
|
||||
const streams = await garnetClient.readGroup(
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
EVAL_REVIEW_GROUP,
|
||||
options.consumerName,
|
||||
">",
|
||||
{
|
||||
blockMs: options.blockMs ?? 5_000,
|
||||
count: 1,
|
||||
},
|
||||
);
|
||||
const message = streams?.[0]?.messages[0];
|
||||
if (!message) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
streamId: message.id,
|
||||
entry: parseReviewQueueEntry(message),
|
||||
};
|
||||
}
|
||||
|
||||
export async function acknowledgeReviewItem(
|
||||
garnetClient: Pick<GarnetClient, "acknowledgeStream">,
|
||||
streamId: string,
|
||||
): Promise<void> {
|
||||
await garnetClient.acknowledgeStream(
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
EVAL_REVIEW_GROUP,
|
||||
streamId,
|
||||
);
|
||||
}
|
||||
|
||||
export async function recordReviewLabel(
|
||||
garnetClient: Pick<GarnetClient, "acknowledgeStream" | "appendStream">,
|
||||
options: {
|
||||
consumerName: string;
|
||||
label: ReviewLabel;
|
||||
streamId: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
{
|
||||
type: "eval-review-label",
|
||||
evalStreamId: options.label.evalStreamId,
|
||||
reviewedAt: options.label.reviewedAt,
|
||||
reviewedBy: options.label.reviewedBy,
|
||||
payload: JSON.stringify(options.label),
|
||||
},
|
||||
{
|
||||
maxLen: getEvalReviewLabelMaxLen(),
|
||||
},
|
||||
);
|
||||
await acknowledgeReviewItem(garnetClient, options.streamId);
|
||||
}
|
||||
|
||||
export async function resetPendingReviewItems(
|
||||
garnetClient: ReviewPendingGarnetClient,
|
||||
options: {
|
||||
consumerName: string;
|
||||
count?: number | undefined;
|
||||
idleMs?: number | undefined;
|
||||
nowMs: number;
|
||||
},
|
||||
): Promise<{ resetCount: number }> {
|
||||
await ensureReviewConsumerGroup(garnetClient);
|
||||
const pending = await garnetClient.getPendingStreamRange(
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
EVAL_REVIEW_GROUP,
|
||||
options.count ?? 100,
|
||||
{
|
||||
idleMs: options.idleMs ?? 60 * 60_000,
|
||||
},
|
||||
);
|
||||
let resetCount = 0;
|
||||
|
||||
for (const entry of pending) {
|
||||
const claimed = await garnetClient.claimPendingStream(
|
||||
EVAL_REVIEW_QUEUE_STREAM_KEY,
|
||||
EVAL_REVIEW_GROUP,
|
||||
options.consumerName,
|
||||
options.idleMs ?? 60 * 60_000,
|
||||
entry.id,
|
||||
);
|
||||
const message = claimed[0];
|
||||
if (!message) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const reviewEntry = parseReviewQueueEntry(message);
|
||||
await enqueueReviewCandidate(garnetClient, {
|
||||
...reviewEntry,
|
||||
enqueuedAt: new Date(options.nowMs).toISOString(),
|
||||
});
|
||||
await acknowledgeReviewItem(garnetClient, message.id);
|
||||
resetCount += 1;
|
||||
} catch {
|
||||
await acknowledgeReviewItem(garnetClient, message.id);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
resetCount,
|
||||
};
|
||||
}
|
||||
|
||||
export async function readReviewLabels(
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
): Promise<ReviewLabel[]> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
"-",
|
||||
"+",
|
||||
);
|
||||
return entries.flatMap((entry) => {
|
||||
try {
|
||||
return [parseReviewLabelEntry(entry)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function routeReviewCandidates(
|
||||
garnetClient: ReviewRouteGarnetClient,
|
||||
options: {
|
||||
nowMs: number;
|
||||
sinceId: string;
|
||||
},
|
||||
): Promise<{ enqueuedCount: number; lastSeenId: string }> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
enqueuedCount: 0,
|
||||
lastSeenId: options.sinceId,
|
||||
};
|
||||
}
|
||||
|
||||
const perCategoryLimit = getEvalReviewPerCategoryLimit();
|
||||
const counts = new Map<ReviewReason, number>();
|
||||
let enqueuedCount = 0;
|
||||
let lastSeenId = options.sinceId;
|
||||
|
||||
for (const entry of entries) {
|
||||
lastSeenId = entry.id;
|
||||
const parsed = parseEvalResult(entry);
|
||||
if (!parsed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const reasons = getCandidateReasons(parsed);
|
||||
if (await isRareSlice(garnetClient, parsed)) {
|
||||
reasons.push("rare-slice");
|
||||
}
|
||||
|
||||
for (const reason of reasons) {
|
||||
const key = reasonLimitKey(reason);
|
||||
if ((counts.get(key) ?? 0) >= perCategoryLimit) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const dedupeKey = `lolympus-agent:eval:review:dedupe:${entry.id}`;
|
||||
const claimed = await garnetClient.setStringIfAbsent(
|
||||
dedupeKey,
|
||||
reason,
|
||||
getEvalReviewDedupeTtlSec(),
|
||||
);
|
||||
if (!claimed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await enqueueReviewCandidate(
|
||||
garnetClient,
|
||||
toQueueEntry(parsed, reason, options.nowMs),
|
||||
);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
enqueuedCount += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await garnetClient.setString(EVAL_REVIEW_CURSOR_KEY, lastSeenId);
|
||||
|
||||
return {
|
||||
enqueuedCount,
|
||||
lastSeenId,
|
||||
};
|
||||
}
|
||||
+57
-3
@@ -14,6 +14,10 @@ import { getLogger } from "@obs/logger";
|
||||
import type { RoomMessage } from "@room/types";
|
||||
import { RoomContextService } from "@room/room-context-service";
|
||||
import { OnboardingSkill } from "@skills/onboarding/index";
|
||||
import {
|
||||
ProcessTraceCollector,
|
||||
runWithProcessTrace,
|
||||
} from "@skills/onboarding/runtime/trace";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier";
|
||||
import { deriveState } from "@skills/onboarding/state/derive";
|
||||
@@ -24,6 +28,7 @@ import type {
|
||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
@@ -182,7 +187,7 @@ function toProviderEndpoints(
|
||||
return urls.map((url) => ({ url, provider }));
|
||||
}
|
||||
|
||||
function createEvalLlmClient(): LlmClient {
|
||||
export function createEvalLlmClient(): LlmClient {
|
||||
const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]);
|
||||
const llamaEndpoints = parseEndpointList(
|
||||
process.env["AGENT_LLM_API_URLS"],
|
||||
@@ -336,6 +341,41 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
|
||||
expected.mustNotInclude = mustNotInclude;
|
||||
}
|
||||
|
||||
const processExpectedRaw = raw["processExpected"];
|
||||
if (isRecord(processExpectedRaw)) {
|
||||
const expectedTools = getStringArrayField(
|
||||
processExpectedRaw,
|
||||
"expectedTools",
|
||||
);
|
||||
const expectedApproach = getStringField(
|
||||
processExpectedRaw,
|
||||
"expectedApproach",
|
||||
);
|
||||
const toolModeRaw = getStringField(processExpectedRaw, "toolMode");
|
||||
const minStepsRaw = processExpectedRaw["minSteps"];
|
||||
const processExpected: NonNullable<EvalCaseExpected["processExpected"]> = {};
|
||||
|
||||
if (expectedTools) {
|
||||
processExpected.expectedTools = expectedTools;
|
||||
}
|
||||
if (expectedApproach) {
|
||||
processExpected.expectedApproach = expectedApproach;
|
||||
}
|
||||
if (toolModeRaw === "exact" || toolModeRaw === "subset") {
|
||||
processExpected.toolMode = toolModeRaw;
|
||||
}
|
||||
if (
|
||||
typeof minStepsRaw === "number" &&
|
||||
Number.isInteger(minStepsRaw) &&
|
||||
minStepsRaw > 0
|
||||
) {
|
||||
processExpected.minSteps = minStepsRaw;
|
||||
}
|
||||
if (Object.keys(processExpected).length > 0) {
|
||||
expected.processExpected = processExpected;
|
||||
}
|
||||
}
|
||||
|
||||
return expected;
|
||||
}
|
||||
|
||||
@@ -605,6 +645,7 @@ async function evaluateCase(
|
||||
noopShadowEvalPublisher,
|
||||
);
|
||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||
const processTrace = new ProcessTraceCollector();
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
@@ -616,7 +657,7 @@ async function evaluateCase(
|
||||
trigger: input.request.trigger,
|
||||
userId: input.request.userId,
|
||||
},
|
||||
async () => {
|
||||
() => runWithProcessTrace(processTrace, async () => {
|
||||
const classified = await classifier.classify(
|
||||
input.request.userMessage,
|
||||
input.request.trigger,
|
||||
@@ -651,6 +692,11 @@ async function evaluateCase(
|
||||
skipped: true as const,
|
||||
reason: "disabled",
|
||||
};
|
||||
const processTraceSnapshot = processTrace.snapshot();
|
||||
const processJudge = judgeProcessRule(
|
||||
evalCase.expected,
|
||||
processTraceSnapshot,
|
||||
);
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
@@ -663,11 +709,14 @@ async function evaluateCase(
|
||||
response,
|
||||
ruleJudge,
|
||||
llmJudge,
|
||||
processJudge,
|
||||
processTrace: processTraceSnapshot,
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
const processTraceSnapshot = processTrace.snapshot();
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
@@ -691,10 +740,15 @@ async function evaluateCase(
|
||||
skipped: true,
|
||||
reason: "skill-error",
|
||||
},
|
||||
processJudge: judgeProcessRule(
|
||||
evalCase.expected,
|
||||
processTraceSnapshot,
|
||||
),
|
||||
processTrace: processTraceSnapshot,
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+24
-18
@@ -5,6 +5,7 @@ import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import {
|
||||
getBackgroundStreamMaxLen,
|
||||
getShadowEvalSamplingRate,
|
||||
@@ -17,6 +18,7 @@ export interface ShadowEvalEnvelope {
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
@@ -52,27 +54,31 @@ export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
|
||||
const logger = getRequestLogger();
|
||||
|
||||
const fields: Record<string, string> = {
|
||||
type: "shadow-eval",
|
||||
requestId: envelope.request.requestId,
|
||||
chatroomId: envelope.request.chatroomId,
|
||||
userId: envelope.request.userId,
|
||||
tier: String(envelope.derived.tier),
|
||||
latencyMs:
|
||||
envelope.latencyMs !== undefined ? String(envelope.latencyMs) : "",
|
||||
input: JSON.stringify({
|
||||
request: envelope.request,
|
||||
}),
|
||||
output: JSON.stringify(envelope.response),
|
||||
derived: JSON.stringify(envelope.derived),
|
||||
classified: JSON.stringify(envelope.classified),
|
||||
createdAt: envelope.createdAt,
|
||||
};
|
||||
|
||||
if (envelope.process) {
|
||||
fields.process = JSON.stringify(envelope.process);
|
||||
}
|
||||
|
||||
void this.garnetClient
|
||||
.appendStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval",
|
||||
requestId: envelope.request.requestId,
|
||||
chatroomId: envelope.request.chatroomId,
|
||||
userId: envelope.request.userId,
|
||||
tier: String(envelope.derived.tier),
|
||||
latencyMs:
|
||||
envelope.latencyMs !== undefined
|
||||
? String(envelope.latencyMs)
|
||||
: "",
|
||||
input: JSON.stringify({
|
||||
request: envelope.request,
|
||||
}),
|
||||
output: JSON.stringify(envelope.response),
|
||||
derived: JSON.stringify(envelope.derived),
|
||||
classified: JSON.stringify(envelope.classified),
|
||||
createdAt: envelope.createdAt,
|
||||
},
|
||||
fields,
|
||||
{
|
||||
maxLen: getBackgroundStreamMaxLen(),
|
||||
},
|
||||
|
||||
+349
-28
@@ -13,22 +13,59 @@ import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdDriftPct,
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getEvalAggTickMs,
|
||||
getEvalArchiveTickMs,
|
||||
getEvalDriftTickMs,
|
||||
getEvalProcessLlmJudgeEnabled,
|
||||
getEvalProcessLlmJudgeSamplingRate,
|
||||
getEvalPromoteTickMs,
|
||||
getEvalReviewTickMs,
|
||||
getEvalStreamMaxLen,
|
||||
} from "../config.js";
|
||||
import {
|
||||
aggregateRecentEvalResults,
|
||||
EVAL_AGG_CURSOR_KEY,
|
||||
sliceKeyToString,
|
||||
} from "./aggregator.js";
|
||||
import {
|
||||
archiveEvalResults,
|
||||
EVAL_ARCHIVE_CURSOR_KEY,
|
||||
type MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { computeDriftReports } from "./drift.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import {
|
||||
judgeProcessRule,
|
||||
judgeProcessWithLlm,
|
||||
} from "./judges/process-judge.js";
|
||||
import {
|
||||
appendCandidatesToJsonl,
|
||||
selectPromotionCandidates,
|
||||
} from "./promote.js";
|
||||
import {
|
||||
EVAL_REVIEW_CURSOR_KEY,
|
||||
routeReviewCandidates,
|
||||
} from "./review/queue.js";
|
||||
import {
|
||||
archiveReviewLabels,
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
} from "./review/label-archive.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
ProcessJudgeResult,
|
||||
} from "./types.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
@@ -57,6 +94,7 @@ interface ShadowEvalJob {
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
@@ -112,6 +150,14 @@ function parseJsonField<T>(
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
@@ -140,8 +186,11 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
|
||||
message.message["process"],
|
||||
);
|
||||
|
||||
return {
|
||||
const job: ShadowEvalJob = {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
@@ -158,6 +207,12 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
"output",
|
||||
),
|
||||
};
|
||||
|
||||
if (process) {
|
||||
job.process = process;
|
||||
}
|
||||
|
||||
return job;
|
||||
}
|
||||
|
||||
export function calculatePercentile(
|
||||
@@ -206,11 +261,18 @@ export function calculateLlmErrorRate(
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private lastAggregateAt = 0;
|
||||
private archiveFailureCount = 0;
|
||||
private lastAlertCheckAt = 0;
|
||||
private lastArchiveAt = 0;
|
||||
private lastDriftAt = 0;
|
||||
private lastEvalAggregateAt = 0;
|
||||
private lastPromoteAt = 0;
|
||||
private lastReviewAt = 0;
|
||||
private loopPromise: Promise<void> | null = null;
|
||||
private readonly logger = getLogger();
|
||||
private readonly recentEvalObservations: EvalObservation[] = [];
|
||||
private stopRequested = false;
|
||||
private warnedMissingMongo = false;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
@@ -218,6 +280,7 @@ export class ShadowWorker {
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly alertSender?: AlertSender,
|
||||
private readonly consumerName = `bg-worker-${process.pid}`,
|
||||
private readonly mongoConnection?: MongoConnectionLike | undefined,
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
@@ -432,6 +495,9 @@ export class ShadowWorker {
|
||||
llmJudge.skipped &&
|
||||
llmJudge.reason !== "empty-response" &&
|
||||
llmJudge.reason !== "disabled";
|
||||
const processJudge = job.process
|
||||
? await this.judgeProcess(job, message.id)
|
||||
: undefined;
|
||||
|
||||
this.recordEvalObservation({
|
||||
llmJudgeErrored,
|
||||
@@ -440,37 +506,81 @@ export class ShadowWorker {
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
});
|
||||
|
||||
const resultFields: Record<string, string> = {
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
phase: job.derived.phase.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
if (job.process) {
|
||||
resultFields.process = JSON.stringify(job.process);
|
||||
}
|
||||
if (processJudge) {
|
||||
resultFields.processJudge = JSON.stringify(processJudge);
|
||||
}
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs:
|
||||
job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall:
|
||||
llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
},
|
||||
resultFields,
|
||||
{
|
||||
maxLen: getEvalStreamMaxLen(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async judgeProcess(
|
||||
job: ShadowEvalJob,
|
||||
messageId: string,
|
||||
): Promise<ProcessJudgeResult> {
|
||||
if (!job.process) {
|
||||
throw new Error("Missing process trace");
|
||||
}
|
||||
|
||||
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
|
||||
if (
|
||||
!getEvalProcessLlmJudgeEnabled() ||
|
||||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
||||
) {
|
||||
return ruleResult;
|
||||
}
|
||||
|
||||
const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, {
|
||||
request: job.request,
|
||||
finalText: job.response.text,
|
||||
});
|
||||
|
||||
return {
|
||||
checks: [...ruleResult.checks, ...llmResult.checks],
|
||||
passedCount: ruleResult.passedCount + llmResult.passedCount,
|
||||
totalCount: ruleResult.totalCount + llmResult.totalCount,
|
||||
score: Number(
|
||||
(
|
||||
(ruleResult.score * ruleResult.totalCount +
|
||||
llmResult.score * llmResult.totalCount) /
|
||||
Math.max(1, ruleResult.totalCount + llmResult.totalCount)
|
||||
).toFixed(3),
|
||||
),
|
||||
rationale: llmResult.rationale,
|
||||
};
|
||||
}
|
||||
|
||||
private async processSummarize(message: StreamEntry): Promise<void> {
|
||||
const chatroomId = message.message["chatroomId"];
|
||||
const userId = message.message["userId"];
|
||||
@@ -563,16 +673,23 @@ export class ShadowWorker {
|
||||
}
|
||||
|
||||
private async runPeriodicChecksIfNeeded(): Promise<void> {
|
||||
const now = Date.now();
|
||||
|
||||
await this.runAggregationTickIfNeeded(now);
|
||||
await this.runArchiveTickIfNeeded(now);
|
||||
await this.runDriftTickIfNeeded(now);
|
||||
await this.runPromoteTickIfNeeded(now);
|
||||
await this.runReviewTickIfNeeded(now);
|
||||
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
|
||||
if (now - this.lastAlertCheckAt < ALERT_TICK_MS) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastAggregateAt = now;
|
||||
this.lastAlertCheckAt = now;
|
||||
this.pruneOldObservations(now);
|
||||
|
||||
await this.checkLatencyAlert(now);
|
||||
@@ -581,6 +698,210 @@ export class ShadowWorker {
|
||||
await this.checkOpenCircuitAlerts(now);
|
||||
}
|
||||
|
||||
private async runAggregationTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastEvalAggregateAt = now;
|
||||
const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY);
|
||||
const result = await aggregateRecentEvalResults(this.garnetClient, {
|
||||
sinceId: sinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
});
|
||||
|
||||
if (result.updatedSlices > 0) {
|
||||
this.logger.debug(
|
||||
{
|
||||
lastSeenId: result.lastSeenId,
|
||||
updatedSlices: result.updatedSlices,
|
||||
},
|
||||
"[ShadowWorker] Aggregated shadow eval results",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private warnMissingMongoOnce(): void {
|
||||
if (this.warnedMissingMongo) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.warnedMissingMongo = true;
|
||||
this.logger.warn(
|
||||
"[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled",
|
||||
);
|
||||
}
|
||||
|
||||
private async runArchiveTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastArchiveAt < getEvalArchiveTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastArchiveAt = now;
|
||||
if (!this.mongoConnection) {
|
||||
this.warnMissingMongoOnce();
|
||||
return;
|
||||
}
|
||||
|
||||
const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY);
|
||||
try {
|
||||
const result = await archiveEvalResults(
|
||||
this.garnetClient,
|
||||
this.mongoConnection,
|
||||
{
|
||||
sinceId: sinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
},
|
||||
);
|
||||
const labelSinceId = await this.garnetClient.getString(
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
);
|
||||
const labelResult = await archiveReviewLabels(
|
||||
this.garnetClient,
|
||||
this.mongoConnection,
|
||||
{
|
||||
sinceId: labelSinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
},
|
||||
);
|
||||
this.archiveFailureCount = 0;
|
||||
if (result.archivedCount > 0) {
|
||||
this.logger.info(
|
||||
{
|
||||
archivedCount: result.archivedCount,
|
||||
lastArchivedId: result.lastArchivedId,
|
||||
},
|
||||
"[ShadowWorker] Archived shadow eval results",
|
||||
);
|
||||
}
|
||||
if (labelResult.archivedCount > 0) {
|
||||
this.logger.info(
|
||||
{
|
||||
archivedCount: labelResult.archivedCount,
|
||||
lastArchivedId: labelResult.lastArchivedId,
|
||||
},
|
||||
"[ShadowWorker] Archived review labels",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.archiveFailureCount += 1;
|
||||
this.logger.warn(
|
||||
{
|
||||
err: error,
|
||||
failureCount: this.archiveFailureCount,
|
||||
},
|
||||
"[ShadowWorker] Failed to archive shadow eval results",
|
||||
);
|
||||
|
||||
if (this.archiveFailureCount >= 3 && this.alertSender) {
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "archive-stalled",
|
||||
metric: "shadow_eval.archive_failures",
|
||||
severity: "warn",
|
||||
suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.",
|
||||
threshold: 3,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: this.archiveFailureCount,
|
||||
window: "consecutive",
|
||||
},
|
||||
{
|
||||
dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled",
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runDriftTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastDriftAt < getEvalDriftTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastDriftAt = now;
|
||||
if (!this.mongoConnection) {
|
||||
this.warnMissingMongoOnce();
|
||||
return;
|
||||
}
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const threshold = getAlertThresholdDriftPct();
|
||||
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
|
||||
const significantReports = reports.filter((report) => report.significant);
|
||||
|
||||
for (const report of significantReports) {
|
||||
const slice = sliceKeyToString(report.slice);
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "eval-drift",
|
||||
metric: "shadow_eval.drift.llm_overall_drop_pct",
|
||||
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: report.dropPct,
|
||||
window: "24h_vs_7d",
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async runPromoteTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastPromoteAt < getEvalPromoteTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastPromoteAt = now;
|
||||
if (!this.mongoConnection) {
|
||||
this.warnMissingMongoOnce();
|
||||
return;
|
||||
}
|
||||
|
||||
const candidates = await selectPromotionCandidates(this.mongoConnection, {
|
||||
nowMs: now,
|
||||
});
|
||||
if (candidates.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await appendCandidatesToJsonl(candidates);
|
||||
this.logger.info(
|
||||
{
|
||||
appendedCount: result.appendedCount,
|
||||
outputPath: result.outputPath,
|
||||
},
|
||||
"[ShadowWorker] Promoted shadow eval candidates",
|
||||
);
|
||||
}
|
||||
|
||||
private async runReviewTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastReviewAt < getEvalReviewTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastReviewAt = now;
|
||||
const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY);
|
||||
const result = await routeReviewCandidates(this.garnetClient, {
|
||||
sinceId: sinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
});
|
||||
|
||||
if (result.enqueuedCount > 0) {
|
||||
this.logger.info(
|
||||
{
|
||||
enqueuedCount: result.enqueuedCount,
|
||||
lastSeenId: result.lastSeenId,
|
||||
},
|
||||
"[ShadowWorker] Routed eval results to review queue",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
|
||||
export interface EvalRoomContextMessage {
|
||||
senderId: string;
|
||||
@@ -26,6 +27,12 @@ export interface EvalCaseExpected {
|
||||
intent?: string | undefined;
|
||||
mustIncludeAny?: string[] | undefined;
|
||||
mustNotInclude?: string[] | undefined;
|
||||
processExpected?: {
|
||||
expectedApproach?: string | undefined;
|
||||
expectedTools?: string[] | undefined;
|
||||
minSteps?: number | undefined;
|
||||
toolMode?: "exact" | "subset" | undefined;
|
||||
} | undefined;
|
||||
responseMode?: string | undefined;
|
||||
}
|
||||
|
||||
@@ -82,6 +89,30 @@ export type LlmJudgeResult =
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export interface ProcessJudgeCheck {
|
||||
detail?: string | undefined;
|
||||
passed: boolean;
|
||||
rule:
|
||||
| "critic-decision-justified"
|
||||
| "intent-match"
|
||||
| "min-steps"
|
||||
| "planner-approach"
|
||||
| "tier-appropriate"
|
||||
| "tool-args-valid"
|
||||
| "tools-selected-correctly"
|
||||
| "validate-no-reject";
|
||||
skipped: boolean;
|
||||
step: string;
|
||||
}
|
||||
|
||||
export interface ProcessJudgeResult {
|
||||
checks: ProcessJudgeCheck[];
|
||||
passedCount: number;
|
||||
rationale?: string | undefined;
|
||||
score: number;
|
||||
totalCount: number;
|
||||
}
|
||||
|
||||
export interface EvalDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
@@ -109,6 +140,8 @@ export interface EvalCaseResult {
|
||||
response: AgentResponse;
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
processTrace?: ProcessTraceSnapshot | undefined;
|
||||
error?: string | undefined;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession, PromptContext } from "../types.js";
|
||||
@@ -10,11 +13,104 @@ import {
|
||||
} from "./layers.js";
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
| "examples"
|
||||
| "persona"
|
||||
| "policy"
|
||||
| "state"
|
||||
| "tool-policy";
|
||||
|
||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||
|
||||
export interface PromptLayerDiff {
|
||||
layer: PromptLayerName;
|
||||
op: PromptLayerOperation;
|
||||
reason?: string | undefined;
|
||||
text?: string | undefined;
|
||||
}
|
||||
|
||||
export interface PromptVariantManifest {
|
||||
description?: string | undefined;
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
|
||||
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
|
||||
return layer === "tool-policy" ? "policy" : layer;
|
||||
}
|
||||
|
||||
function applyLayerDiff(
|
||||
layers: Record<Exclude<PromptLayerName, "tool-policy">, string>,
|
||||
diff: PromptLayerDiff,
|
||||
): void {
|
||||
const target = getLayerTarget(diff.layer);
|
||||
|
||||
if (diff.op === "remove") {
|
||||
layers[target] = "";
|
||||
return;
|
||||
}
|
||||
|
||||
const text = diff.text?.trim();
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (diff.op === "replace") {
|
||||
layers[target] = text;
|
||||
return;
|
||||
}
|
||||
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
export function loadPromptVariantManifest(
|
||||
variantId: string,
|
||||
): PromptVariantManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (variantCache.has(normalized)) {
|
||||
return variantCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
variantCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class PromptBuilder {
|
||||
private constructor(
|
||||
private readonly derived: DerivedState,
|
||||
private readonly session: OnboardingSession | null,
|
||||
private readonly context: PromptContext,
|
||||
private readonly variant?: PromptVariantManifest | undefined,
|
||||
) {}
|
||||
|
||||
static forDerived(
|
||||
@@ -25,7 +121,18 @@ export class PromptBuilder {
|
||||
return new PromptBuilder(derived, session, context);
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
applyVariant(variant: PromptVariantManifest | string): PromptBuilder {
|
||||
const manifest =
|
||||
typeof variant === "string" ? loadPromptVariantManifest(variant) : variant;
|
||||
return new PromptBuilder(
|
||||
this.derived,
|
||||
this.session,
|
||||
this.context,
|
||||
manifest ?? undefined,
|
||||
);
|
||||
}
|
||||
|
||||
buildSystemPrompt(): string {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
: this.context.conversationHistory;
|
||||
@@ -34,21 +141,41 @@ export class PromptBuilder {
|
||||
conversationHistory: promptHistory,
|
||||
};
|
||||
|
||||
const systemPrompt = [
|
||||
personaLayer(),
|
||||
policyLayer(this.derived.intent, {
|
||||
const layers = {
|
||||
persona: personaLayer(),
|
||||
policy: policyLayer(this.derived.intent, {
|
||||
toolPolicy:
|
||||
this.derived.responseMode === "llm-with-tools"
|
||||
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
|
||||
: undefined,
|
||||
}),
|
||||
stateLayer(this.session, this.derived),
|
||||
examplesLayer(this.derived.intent),
|
||||
contextLayer(promptContext),
|
||||
].join("\n\n");
|
||||
state: stateLayer(this.session, this.derived),
|
||||
examples: examplesLayer(this.derived.intent),
|
||||
context: contextLayer(promptContext),
|
||||
};
|
||||
|
||||
for (const diff of this.variant?.diff ?? []) {
|
||||
applyLayerDiff(layers, diff);
|
||||
}
|
||||
|
||||
return [
|
||||
{ role: "system", content: systemPrompt },
|
||||
layers.persona,
|
||||
layers.policy,
|
||||
layers.state,
|
||||
layers.examples,
|
||||
layers.context,
|
||||
]
|
||||
.filter((layer) => layer.trim().length > 0)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
: this.context.conversationHistory;
|
||||
|
||||
return [
|
||||
{ role: "system", content: this.buildSystemPrompt() },
|
||||
...promptHistory.map((message) => ({
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
getTier2CriticModel,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface CritiqueResult {
|
||||
changed: boolean;
|
||||
@@ -55,6 +56,7 @@ export class OnboardingCritic {
|
||||
messages: ChatMessage[],
|
||||
options?: { deadlineMs?: number | undefined },
|
||||
): Promise<CritiqueResult> {
|
||||
const traceStep = getActiveProcessTrace()?.startStep("critic");
|
||||
const response = await this.llmClient.chatCompletion(
|
||||
buildCriticMessages(draft, derived, messages),
|
||||
{
|
||||
@@ -69,12 +71,20 @@ export class OnboardingCritic {
|
||||
|
||||
const text = response?.content.trim() ?? "";
|
||||
if (!text || text.startsWith("[OK]")) {
|
||||
traceStep?.finish({
|
||||
decision: "keep",
|
||||
rationale: text || "empty critic response",
|
||||
});
|
||||
return {
|
||||
changed: false,
|
||||
text: draft,
|
||||
};
|
||||
}
|
||||
|
||||
traceStep?.finish({
|
||||
decision: "rewrite",
|
||||
rationale: text.slice(0, 200),
|
||||
});
|
||||
return {
|
||||
changed: true,
|
||||
text,
|
||||
|
||||
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const logger = getRequestLogger();
|
||||
const traceStep = getActiveProcessTrace()?.startStep("plan");
|
||||
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
|
||||
traceStep?.finish({
|
||||
approach: plan.approach,
|
||||
mustAvoid: plan.mustAvoid,
|
||||
mustInclude: plan.mustInclude,
|
||||
suggestedTools: plan.suggestedTools,
|
||||
targetLengthChars: plan.targetLengthChars,
|
||||
});
|
||||
return plan;
|
||||
};
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return firstPlan;
|
||||
return finishTrace(firstPlan);
|
||||
}
|
||||
|
||||
const retryMessages = [
|
||||
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return retryPlan;
|
||||
return finishTrace(retryPlan);
|
||||
}
|
||||
|
||||
const fallbackPlan = getDefaultPlan(derived);
|
||||
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner fell back to default plan",
|
||||
);
|
||||
return fallbackPlan;
|
||||
return finishTrace(fallbackPlan);
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface ToolLoopOptions {
|
||||
deadlineMs?: number | undefined;
|
||||
@@ -107,15 +108,33 @@ export async function runWithTools(
|
||||
"[ToolLoop] Executing tool call",
|
||||
);
|
||||
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||
let result: unknown;
|
||||
try {
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: true,
|
||||
resultSummary: summarizeToolResult(result),
|
||||
});
|
||||
} catch (error) {
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
@@ -137,3 +156,11 @@ export async function runWithTools(
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function summarizeToolResult(result: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(result).slice(0, 200);
|
||||
} catch {
|
||||
return String(result).slice(0, 200);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
export type ProcessTraceStepName =
|
||||
| "classify"
|
||||
| "critic"
|
||||
| "derive"
|
||||
| "plan"
|
||||
| "respond"
|
||||
| "tool-call"
|
||||
| "validate";
|
||||
|
||||
export interface ProcessTraceStep<T> {
|
||||
name: ProcessTraceStepName;
|
||||
startedAtMs: number;
|
||||
durationMs: number;
|
||||
output: T;
|
||||
}
|
||||
|
||||
export interface ProcessTraceSnapshot {
|
||||
steps: {
|
||||
classify?: ProcessTraceStep<{
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: "regex" | "llm";
|
||||
}> | undefined;
|
||||
derive?: ProcessTraceStep<{
|
||||
phase: string;
|
||||
responseMode: string;
|
||||
shouldRespond: boolean;
|
||||
tier: 0 | 1 | 2;
|
||||
}> | undefined;
|
||||
plan?: ProcessTraceStep<{
|
||||
approach: string;
|
||||
mustAvoid: string[];
|
||||
mustInclude: string[];
|
||||
suggestedTools: string[];
|
||||
targetLengthChars: number;
|
||||
}> | undefined;
|
||||
toolCalls?: Array<
|
||||
ProcessTraceStep<{
|
||||
args: unknown;
|
||||
name: string;
|
||||
ok: boolean;
|
||||
resultSummary: string;
|
||||
}>
|
||||
> | undefined;
|
||||
respond?: ProcessTraceStep<{
|
||||
draftText: string;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
}> | undefined;
|
||||
critic?: ProcessTraceStep<{
|
||||
decision: "keep" | "rewrite";
|
||||
rationale: string;
|
||||
}> | undefined;
|
||||
validate?: ProcessTraceStep<{
|
||||
reasons: string[];
|
||||
status: "pass" | "reject" | "soft-warn";
|
||||
}> | undefined;
|
||||
};
|
||||
}
|
||||
|
||||
type ProcessTraceOutputByName = {
|
||||
classify: NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"];
|
||||
critic: NonNullable<ProcessTraceSnapshot["steps"]["critic"]>["output"];
|
||||
derive: NonNullable<ProcessTraceSnapshot["steps"]["derive"]>["output"];
|
||||
plan: NonNullable<ProcessTraceSnapshot["steps"]["plan"]>["output"];
|
||||
respond: NonNullable<ProcessTraceSnapshot["steps"]["respond"]>["output"];
|
||||
"tool-call": NonNullable<
|
||||
NonNullable<ProcessTraceSnapshot["steps"]["toolCalls"]>[number]
|
||||
>["output"];
|
||||
validate: NonNullable<ProcessTraceSnapshot["steps"]["validate"]>["output"];
|
||||
};
|
||||
|
||||
const processTraceStorage = new AsyncLocalStorage<ProcessTraceCollector>();
|
||||
|
||||
export class ProcessTraceCollector {
|
||||
private readonly steps: ProcessTraceSnapshot["steps"] = {};
|
||||
|
||||
startStep<Name extends ProcessTraceStepName>(
|
||||
name: Name,
|
||||
): {
|
||||
finish: (output: ProcessTraceOutputByName[Name]) => void;
|
||||
} {
|
||||
const startedAtMs = Date.now();
|
||||
let finished = false;
|
||||
|
||||
return {
|
||||
finish: (output: ProcessTraceOutputByName[Name]): void => {
|
||||
if (finished) {
|
||||
return;
|
||||
}
|
||||
finished = true;
|
||||
|
||||
const step: ProcessTraceStep<ProcessTraceOutputByName[Name]> = {
|
||||
name,
|
||||
startedAtMs,
|
||||
durationMs: Math.max(0, Date.now() - startedAtMs),
|
||||
output,
|
||||
};
|
||||
|
||||
if (name === "tool-call") {
|
||||
this.steps.toolCalls = [
|
||||
...(this.steps.toolCalls ?? []),
|
||||
step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["tool-call"]
|
||||
>,
|
||||
];
|
||||
return;
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case "classify":
|
||||
this.steps.classify = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["classify"]
|
||||
>;
|
||||
return;
|
||||
case "critic":
|
||||
this.steps.critic = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["critic"]
|
||||
>;
|
||||
return;
|
||||
case "derive":
|
||||
this.steps.derive = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["derive"]
|
||||
>;
|
||||
return;
|
||||
case "plan":
|
||||
this.steps.plan = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["plan"]
|
||||
>;
|
||||
return;
|
||||
case "respond":
|
||||
this.steps.respond = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["respond"]
|
||||
>;
|
||||
return;
|
||||
case "validate":
|
||||
this.steps.validate = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["validate"]
|
||||
>;
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
...this.steps,
|
||||
toolCalls: this.steps.toolCalls
|
||||
? [...this.steps.toolCalls]
|
||||
: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function runWithProcessTrace<T>(
|
||||
collector: ProcessTraceCollector,
|
||||
fn: () => T,
|
||||
): T {
|
||||
return processTraceStorage.run(collector, fn);
|
||||
}
|
||||
|
||||
export function getActiveProcessTrace(): ProcessTraceCollector | undefined {
|
||||
return processTraceStorage.getStore();
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { getMaxResponseLength } from "../../../config.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/iu,
|
||||
@@ -118,6 +119,10 @@ function markValidation(result: ValidationResult): ValidationResult {
|
||||
"agent.validator.status": result.status,
|
||||
"agent.validator.warnings": result.reasons,
|
||||
});
|
||||
getActiveProcessTrace()?.startStep("validate").finish({
|
||||
status: result.status,
|
||||
reasons: result.reasons,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ export function collectNicknameAliases(
|
||||
|
||||
for (const source of sources) {
|
||||
if (Array.isArray(source)) {
|
||||
for (const value of source) {
|
||||
for (const value of source as readonly string[]) {
|
||||
pushAlias(value);
|
||||
}
|
||||
continue;
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { ChatMessage } from '@llm/types';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import type { Skill } from '@skills/skill';
|
||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||
import { createBuiltInToolRegistry } from '@tools/builtins/index';
|
||||
import type { Tool, ToolContext } from '@tools/tool';
|
||||
import { PromptBuilder } from './prompts/builder.js';
|
||||
@@ -27,12 +28,21 @@ import {
|
||||
getRemainingStatusMessage,
|
||||
getTimeoutWarningMessage,
|
||||
} from './onboarding-templates.js';
|
||||
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from '../../config.js';
|
||||
import {
|
||||
getOnboardingSessionKeyPrefix,
|
||||
getLlmTemperature,
|
||||
getPromptVariantId,
|
||||
} from '../../config.js';
|
||||
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
||||
import { deriveState } from './state/derive.js';
|
||||
import { OnboardingIntentClassifier } from './state/classifier.js';
|
||||
import type { DerivedState, Intent } from './state/onboarding-state.js';
|
||||
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
|
||||
import {
|
||||
getActiveProcessTrace,
|
||||
ProcessTraceCollector,
|
||||
runWithProcessTrace,
|
||||
} from './runtime/trace.js';
|
||||
import { validateOnboardingResponse } from './runtime/validator.js';
|
||||
import { normalizeOnboardingSession } from './session.js';
|
||||
import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js';
|
||||
@@ -71,19 +81,57 @@ export class OnboardingSkill implements Skill {
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
if (getActiveProcessTrace()) {
|
||||
return this.handleWithTrace(request, conversationHistory);
|
||||
}
|
||||
|
||||
const collector = new ProcessTraceCollector();
|
||||
return runWithProcessTrace(collector, () =>
|
||||
this.handleWithTrace(request, conversationHistory),
|
||||
);
|
||||
}
|
||||
|
||||
private async handleWithTrace(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const logger = getRequestLogger();
|
||||
const trace = getActiveProcessTrace();
|
||||
const session = await this.loadSession(request.chatroomId, request.userId);
|
||||
|
||||
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
|
||||
return this.makeEmptyResponse(request.requestId);
|
||||
}
|
||||
|
||||
const classifyStep = trace?.startStep('classify');
|
||||
const classified = await this.classifier.classify(request.userMessage, request.trigger);
|
||||
classifyStep?.finish({
|
||||
confidence: classified.confidence,
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
});
|
||||
|
||||
const deriveStep = trace?.startStep('derive');
|
||||
const derived = deriveState(session, request.trigger, classified.intent);
|
||||
deriveStep?.finish({
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
if (!derived.shouldRespond) {
|
||||
return this.makeEmptyResponse(request.requestId);
|
||||
const response = this.makeEmptyResponse(request.requestId);
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse('[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: true,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
let response: AgentResponse;
|
||||
@@ -102,6 +150,15 @@ export class OnboardingSkill implements Skill {
|
||||
}
|
||||
}
|
||||
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
});
|
||||
|
||||
if (derived.intent.kind === 'discord-verify-question') {
|
||||
logger.debug(
|
||||
{
|
||||
@@ -122,6 +179,7 @@ export class OnboardingSkill implements Skill {
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
@@ -185,7 +243,9 @@ export class OnboardingSkill implements Skill {
|
||||
userMessage: this.buildPromptUserMessage(request, derived),
|
||||
};
|
||||
|
||||
return PromptBuilder.forDerived(derived, session, context).build();
|
||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||
const promptVariantId = getPromptVariantId();
|
||||
return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build();
|
||||
}
|
||||
|
||||
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js";
|
||||
import { analyzeNicknameIssues } from "../onboarding-templates.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js";
|
||||
import type { OnboardingPhase } from "./onboarding-state.js";
|
||||
import {
|
||||
getRemainingSteps,
|
||||
isOnboardingCompleted,
|
||||
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
isStep1Completed,
|
||||
} from "@skills/onboarding/state/progress";
|
||||
import type {
|
||||
OnboardingSession,
|
||||
StoredOnboardingSession,
|
||||
} from "@skills/onboarding/types";
|
||||
import type { Tool } from "../tool.js";
|
||||
|
||||
@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { verifyDiscordNicknameTool } from "./verify-discord-nickname.js";
|
||||
import type { OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { DiscordMemberLookupResponse } from "@common/nats-subjects";
|
||||
import { KAKAOTALK_NICKNAME_FORMAT_EXAMPLE } from "@skills/onboarding/constants";
|
||||
|
||||
function makeSession(
|
||||
overrides?: Partial<OnboardingSession>,
|
||||
@@ -35,7 +36,7 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
);
|
||||
|
||||
expect(result.status).toBe("invalid-nickname-format");
|
||||
expect(result.formatExample).toBe("홍길동/95/남/G");
|
||||
expect(result.formatExample).toBe(KAKAOTALK_NICKNAME_FORMAT_EXAMPLE);
|
||||
expect(result.issueMessages?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
@@ -56,11 +57,11 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
joinedAfterMatches: [],
|
||||
};
|
||||
const request = vi.fn(
|
||||
async <TRequest = unknown, TResponse = unknown>(
|
||||
<TRequest = unknown, TResponse = unknown>(
|
||||
_subject: string,
|
||||
_payload: TRequest,
|
||||
_options?: { timeoutMs?: number },
|
||||
): Promise<TResponse> => response as TResponse,
|
||||
): Promise<TResponse> => Promise.resolve(response as TResponse),
|
||||
);
|
||||
const session = makeSession();
|
||||
const result = await verifyDiscordNicknameTool.execute(
|
||||
@@ -102,11 +103,11 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
joinedAfterMatches: [],
|
||||
};
|
||||
const request = vi.fn(
|
||||
async <TRequest = unknown, TResponse = unknown>(
|
||||
<TRequest = unknown, TResponse = unknown>(
|
||||
_subject: string,
|
||||
_payload: TRequest,
|
||||
_options?: { timeoutMs?: number },
|
||||
): Promise<TResponse> => response as TResponse,
|
||||
): Promise<TResponse> => Promise.resolve(response as TResponse),
|
||||
);
|
||||
|
||||
const result = await verifyDiscordNicknameTool.execute(
|
||||
@@ -141,11 +142,11 @@ describe("verifyDiscordNicknameTool", () => {
|
||||
],
|
||||
};
|
||||
const request = vi.fn(
|
||||
async <TRequest = unknown, TResponse = unknown>(
|
||||
<TRequest = unknown, TResponse = unknown>(
|
||||
_subject: string,
|
||||
_payload: TRequest,
|
||||
_options?: { timeoutMs?: number },
|
||||
): Promise<TResponse> => response as TResponse,
|
||||
): Promise<TResponse> => Promise.resolve(response as TResponse),
|
||||
);
|
||||
|
||||
const result = await verifyDiscordNicknameTool.execute(
|
||||
|
||||
@@ -21,6 +21,7 @@ const { getRawSecret } = await import("../common/secret-manager.js");
|
||||
const { ConversationStore } = await import("../conversation/index.js");
|
||||
const { ShadowWorker } = await import("../eval/shadow-worker.js");
|
||||
const { AlertService } = await import("../observability/alert.js");
|
||||
const { initMongoDB, closeMongoDB } = await import("../common/mongodb.js");
|
||||
|
||||
function firstNonEmpty(
|
||||
...values: Array<string | null | undefined>
|
||||
@@ -175,15 +176,30 @@ async function bootstrap(): Promise<void> {
|
||||
});
|
||||
const conversationStore = new ConversationStore(garnetClient);
|
||||
const alertService = new AlertService(garnetClient, natsClient);
|
||||
const mongoConnectionString = firstNonEmpty(
|
||||
getRawSecret("MONGODB_CONNECTION_STRING"),
|
||||
process.env["MONGODB_CONNECTION_STRING"],
|
||||
);
|
||||
const mongoConnection = mongoConnectionString
|
||||
? await initMongoDB(mongoConnectionString)
|
||||
: undefined;
|
||||
if (!mongoConnection) {
|
||||
logger.warn(
|
||||
"[Background] MongoDB connection string missing; eval archive/drift/promote disabled",
|
||||
);
|
||||
}
|
||||
const worker = new ShadowWorker(
|
||||
garnetClient,
|
||||
llmClient,
|
||||
conversationStore,
|
||||
alertService,
|
||||
undefined,
|
||||
mongoConnection,
|
||||
);
|
||||
|
||||
installShutdownHooks(async () => {
|
||||
await worker.stop();
|
||||
await closeMongoDB();
|
||||
await natsClient.close();
|
||||
await garnetClient.close();
|
||||
await shutdownTracing();
|
||||
|
||||
Reference in new issue
Block a user