diff --git a/apps/web/src/app/api/evaluators/[agentId]/export-data/route.ts b/apps/web/src/app/api/evaluators/[agentId]/export-data/route.ts index 214f70caa..21a9e567c 100644 --- a/apps/web/src/app/api/evaluators/[agentId]/export-data/route.ts +++ b/apps/web/src/app/api/evaluators/[agentId]/export-data/route.ts @@ -233,7 +233,7 @@ export async function GET( comment_count: evalVersion.comments.length, comments: evalVersion.comments.map((comment) => ({ header: comment.header, - level: comment.level, + variant: comment.variant, source: comment.source, metadata: comment.metadata, description: comment.description, diff --git a/apps/web/src/app/docs/[docId]/export/ExportClient.tsx b/apps/web/src/app/docs/[docId]/export/ExportClient.tsx index 8537680da..cfda068da 100644 --- a/apps/web/src/app/docs/[docId]/export/ExportClient.tsx +++ b/apps/web/src/app/docs/[docId]/export/ExportClient.tsx @@ -4,6 +4,7 @@ import { useEffect, useState } from "react"; import yaml from "js-yaml"; import { RefreshCw } from "lucide-react"; +import type { CommentVariant } from "@roast/ai"; import CodeBlock from "@/components/CodeBlock"; import { Button } from "@/components/ui/button"; @@ -13,7 +14,7 @@ interface Comment { description: string; importance?: number | null; grade?: number | null; - level?: string | null; + variant?: CommentVariant | null; source?: string | null; metadata?: any; highlight?: { @@ -181,7 +182,7 @@ export function ExportClient({ document, evaluations }: ExportClientProps) { // Add expanded data only if mode is 'expanded' if (commentsMode === 'expanded') { - if (comment.level) commentData.level = comment.level; + if (comment.variant) commentData.variant = comment.variant; if (comment.source) commentData.source = comment.source; if (comment.metadata) commentData.metadata = comment.metadata; diff --git a/apps/web/src/app/docs/[docId]/export/page.tsx b/apps/web/src/app/docs/[docId]/export/page.tsx index 4182dcda8..db81a2df7 100644 --- a/apps/web/src/app/docs/[docId]/export/page.tsx +++ b/apps/web/src/app/docs/[docId]/export/page.tsx @@ -105,7 +105,7 @@ export default async function DocumentExportPage({ description: comment.description, importance: comment.importance, grade: comment.grade, - level: comment.level, + variant: comment.variant, source: comment.source, metadata: comment.metadata, highlight: comment.highlight diff --git a/apps/web/src/app/monitor/evals/page.tsx b/apps/web/src/app/monitor/evals/page.tsx index e60f34f27..7a1dc6ce0 100644 --- a/apps/web/src/app/monitor/evals/page.tsx +++ b/apps/web/src/app/monitor/evals/page.tsx @@ -10,7 +10,7 @@ import { } from "@heroicons/react/24/outline"; import { GradeBadge } from "@/components/GradeBadge"; import { EvaluationContent } from "@/components/evaluation"; - +import type { CommentVariant } from "@roast/ai"; interface Evaluation { id: string; createdAt: string; @@ -45,7 +45,7 @@ interface Evaluation { importance: number | null; grade: number | null; header: string | null; - level: string | null; + variant: string | null; source: string | null; metadata: Record | null; }>; @@ -277,7 +277,7 @@ export default function EvaluationsMonitorPage() { evaluationVersionId: selectedVersion.id, highlightId: comment.id, header: comment.header ?? null, - level: comment.level ?? null, + variant: comment.variant as CommentVariant | null, source: comment.source ?? null, metadata: comment.metadata ?? null, highlight: { diff --git a/apps/web/src/app/tools/utils/toolExamples.ts b/apps/web/src/app/tools/utils/toolExamples.ts index d285882d3..61c692e94 100644 --- a/apps/web/src/app/tools/utils/toolExamples.ts +++ b/apps/web/src/app/tools/utils/toolExamples.ts @@ -362,7 +362,7 @@ export const toolExamples: Record = { index: 0, header: 'Cherry-picked timeframe', description: 'Selecting only the best-performing time period to make an investment look better than it actually is.', - level: 'warning', + variant: 'warning', importance: 85, quotedText: 'grown 1000% in the last year' }, @@ -370,7 +370,7 @@ export const toolExamples: Record = { index: 1, header: 'Survivorship bias', description: 'Only considering successful cases while ignoring all the people who lost money.', - level: 'warning', + variant: 'warning', importance: 80, quotedText: 'Everyone who invested got rich' }, @@ -378,7 +378,7 @@ export const toolExamples: Record = { index: 2, header: 'Survivorship bias again', description: 'Another instance of only showing winners, not losers.', - level: 'warning', + variant: 'warning', importance: 75, quotedText: 'My friend made millions' }, @@ -386,7 +386,7 @@ export const toolExamples: Record = { index: 3, header: 'Hasty generalization', description: 'Drawing broad conclusions from a single example.', - level: 'warning', + variant: 'warning', importance: 70, quotedText: 'My friend made millions, so you will too' } @@ -403,7 +403,7 @@ export const toolExamples: Record = { index: 0, header: 'Vague reference', description: 'Which studies? What did they find? This needs specific citations.', - level: 'warning', + variant: 'warning', importance: 90, quotedText: 'Studies show' }, @@ -411,7 +411,7 @@ export const toolExamples: Record = { index: 1, header: 'Weasel words', description: 'Some research is very vague - how much? What quality?', - level: 'nitpick', + variant: 'nitpick', importance: 40, quotedText: 'Some research' }, @@ -419,7 +419,7 @@ export const toolExamples: Record = { index: 2, header: 'Statement is factual', description: 'This is actually accurate and verifiable.', - level: 'info', + variant: 'info', importance: 20, quotedText: 'contains antioxidants' } @@ -436,7 +436,7 @@ export const toolExamples: Record = { index: 0, header: 'Impossible claim', description: 'No medical treatment has 100% efficacy. This is a red flag for misinformation.', - level: 'error', + variant: 'error', importance: 95, quotedText: 'works 100% of the time' }, @@ -444,7 +444,7 @@ export const toolExamples: Record = { index: 1, header: 'Appeal to conspiracy', description: 'Doctors hate it suggests a conspiracy theory framing that undermines credibility.', - level: 'warning', + variant: 'warning', importance: 85, quotedText: 'Doctors hate it' } diff --git a/apps/web/src/application/workflows/evaluation/exportXml.ts b/apps/web/src/application/workflows/evaluation/exportXml.ts index 2efccfe95..f59ed17e9 100644 --- a/apps/web/src/application/workflows/evaluation/exportXml.ts +++ b/apps/web/src/application/workflows/evaluation/exportXml.ts @@ -1,4 +1,5 @@ import { logger } from "@/infrastructure/logging/logger"; +import type { CommentVariant } from "@roast/ai"; interface ExportEvaluationData { evaluation: { @@ -22,7 +23,7 @@ interface ExportEvaluationData { importance?: number | null; grade?: number | null; header?: string | null; - level?: string | null; + variant?: CommentVariant | null; source?: string | null; metadata?: Record | null; }>; @@ -122,8 +123,8 @@ export function exportEvaluationToXml(data: ExportEvaluationData): string { if (comment.header) { xml += `
\n`; } - if (comment.level) { - xml += ` ${comment.level}\n`; + if (comment.variant) { + xml += ` ${comment.variant}\n`; } if (comment.source) { xml += ` ${comment.source}\n`; diff --git a/apps/web/src/application/workflows/evaluation/types.ts b/apps/web/src/application/workflows/evaluation/types.ts index f4979756b..4960bde6d 100644 --- a/apps/web/src/application/workflows/evaluation/types.ts +++ b/apps/web/src/application/workflows/evaluation/types.ts @@ -1,3 +1,5 @@ +import type { CommentVariant } from "@roast/ai"; + /** * Standardized evaluation display data structure */ @@ -15,7 +17,7 @@ export interface EvaluationDisplayData { evaluationVersionId: string; highlightId: string; header?: string | null; - level?: string | null; + variant?: CommentVariant | null; source?: string | null; metadata?: Record | null; highlight: { @@ -73,7 +75,7 @@ export interface EvaluationContentProps { evaluationVersionId: string; highlightId: string; header?: string | null; - level?: string | null; + variant?: CommentVariant | null; source?: string | null; metadata?: Record | null; highlight: { diff --git a/apps/web/src/components/AgentDetail/tabs/EvaluationsTab.tsx b/apps/web/src/components/AgentDetail/tabs/EvaluationsTab.tsx index f09dee5ce..ea70555e3 100644 --- a/apps/web/src/components/AgentDetail/tabs/EvaluationsTab.tsx +++ b/apps/web/src/components/AgentDetail/tabs/EvaluationsTab.tsx @@ -255,7 +255,7 @@ export function EvaluationsTab({ evaluationVersionId: selectedEvaluation.evaluationId, highlightId: comment.id, header: comment.header ?? null, - level: comment.level ?? null, + variant: comment.variant ?? null, source: comment.source ?? null, metadata: comment.metadata ?? null, highlight: { diff --git a/apps/web/src/components/AgentDetail/types.ts b/apps/web/src/components/AgentDetail/types.ts index 6963d709a..00f0128e3 100644 --- a/apps/web/src/components/AgentDetail/types.ts +++ b/apps/web/src/components/AgentDetail/types.ts @@ -1,4 +1,4 @@ -import type { Agent } from "@roast/ai"; +import type { Agent, CommentVariant } from "@roast/ai"; export interface AgentDetailProps { agent: Agent; @@ -48,7 +48,7 @@ export interface AgentEvaluation { importance?: number | null; grade?: number | null; header?: string | null; - level?: string | null; + variant?: CommentVariant | null; source?: string | null; metadata?: any | null; }>; diff --git a/apps/web/src/components/CommentVariantIndicator.tsx b/apps/web/src/components/CommentVariantIndicator.tsx new file mode 100644 index 000000000..c404f51c7 --- /dev/null +++ b/apps/web/src/components/CommentVariantIndicator.tsx @@ -0,0 +1,147 @@ +import type { CommentVariant } from "@roast/ai"; +import { CheckCircleIcon, CheckIcon, XMarkIcon } from "@heroicons/react/24/solid"; + +interface CommentVariantIndicatorProps { + variant: CommentVariant | null | undefined; + size?: "sm" | "md" | "lg"; + isHovered?: boolean; + className?: string; +} + +/** + * Displays a visual indicator (icon with colored background) for a comment variant + * + * Sizes: + * - sm: 5x5 (h-5 w-5) - used in comment lists + * - md: 6x6 (h-6 w-6) - used in modals + * - lg: custom - can be styled with className + */ +export function CommentVariantIndicator({ + variant, + size = "md", + isHovered = false, + className, +}: CommentVariantIndicatorProps) { + const variantType = variant || "info"; // Default to info if not set + + let bgColor = "bg-blue-500"; + let iconContent: React.ReactNode; + + // Determine background color based on variant and hover state + switch (variantType) { + case "error": + bgColor = isHovered ? "bg-red-500" : "bg-red-400"; + iconContent = ; + break; + case "warning": + bgColor = isHovered ? "bg-orange-500" : "bg-orange-400"; + iconContent = !; + break; + case "nitpick": + bgColor = isHovered ? "bg-fuchsia-500" : "bg-fuchsia-300"; + iconContent = Ā·; + break; + case "success": + bgColor = isHovered ? "bg-green-500" : "bg-green-300"; + iconContent = ; + break; + case "debug": + bgColor = isHovered ? "bg-gray-500" : "bg-gray-400"; + iconContent = d; + break; + case "info": + default: + bgColor = isHovered ? "bg-blue-500" : "bg-blue-400"; + iconContent = i; + break; + } + + // Determine size classes + let sizeClasses = ""; + let iconSizeClass = ""; + let textSizeClass = ""; + + switch (size) { + case "sm": + sizeClasses = "h-5 w-5"; + iconSizeClass = "h-3.5 w-3.5"; + textSizeClass = "text-xs"; + break; + case "md": + sizeClasses = "h-6 w-6"; + iconSizeClass = "h-5 w-5"; + textSizeClass = "text-base"; + break; + case "lg": + sizeClasses = "h-8 w-8"; + iconSizeClass = "h-6 w-6"; + textSizeClass = "text-lg"; + break; + } + + // Clone icon element with size class if it's an icon component + let content = iconContent; + if (typeof iconContent === 'object' && iconContent && 'type' in iconContent) { + const IconComponent = iconContent.type as any; + if (IconComponent === XMarkIcon || IconComponent === CheckIcon || IconComponent === CheckCircleIcon) { + content = ; + } else if (iconContent.type === 'span') { + // For text content, add text size + content = {(iconContent as any).props.children}; + } + } + + return ( +
+ {content} +
+ ); +} + +/** + * Variant for EvaluationComments - uses CheckCircleIcon for success + */ +export function CommentVariantIndicatorCompact({ + variant, + className, +}: Omit) { + const variantType = variant || "info"; + + let bgColor = "bg-blue-400"; + let content: React.ReactNode; + + switch (variantType) { + case "error": + bgColor = "bg-red-500"; + content = ; + break; + case "warning": + bgColor = "bg-amber-500"; + content = !; + break; + case "nitpick": + bgColor = "bg-fuchsia-500"; + content = Ā·; + break; + case "success": + bgColor = "bg-green-500"; + content = ; + break; + case "info": + case "debug": + default: + bgColor = "bg-blue-500"; + content = i; + break; + } + + return ( +
+ {content} +
+ ); +} + diff --git a/apps/web/src/components/DocumentWithEvaluations/components/CommentModal.tsx b/apps/web/src/components/DocumentWithEvaluations/components/CommentModal.tsx index 14e07b4b4..641c85759 100644 --- a/apps/web/src/components/DocumentWithEvaluations/components/CommentModal.tsx +++ b/apps/web/src/components/DocumentWithEvaluations/components/CommentModal.tsx @@ -15,13 +15,12 @@ import { import { commentToYaml } from "@/shared/utils/commentToYaml"; import { parseColoredText } from "@/shared/utils/ui/coloredText"; import { - CheckIcon, ChevronDownIcon, ChevronRightIcon, DocumentDuplicateIcon, - XMarkIcon, } from "@heroicons/react/24/outline"; import type { Comment } from "@roast/ai"; +import { CommentVariantIndicator } from "@/components/CommentVariantIndicator"; import { MARKDOWN_COMPONENTS, MARKDOWN_PLUGINS } from "../config/markdown"; @@ -36,43 +35,6 @@ interface CommentModalProps { onNavigate?: (direction: "prev" | "next") => void; } -function getLevelIndicator(level: string) { - let bgColor = "bg-blue-500"; - let content: React.ReactNode; - - switch (level) { - case "error": - bgColor = "bg-red-500"; - content = ; - break; - case "warning": - bgColor = "bg-amber-500"; - content = ( - ! - ); - break; - case "success": - bgColor = "bg-green-500"; - content = ; - break; - case "info": - default: - bgColor = "bg-blue-500"; - content = ( - i - ); - break; - } - - return ( -
- {content} -
- ); -} - export function CommentModal({ comment, agentName, @@ -129,7 +91,7 @@ export function CommentModal({ if (!comment) return null; - const level = comment.level || "info"; + const variant = comment.variant || "info"; const handleCopy = async () => { try { @@ -209,7 +171,7 @@ export function CommentModal({
- {getLevelIndicator(level)} +
{comment.header ? ( diff --git a/apps/web/src/components/DocumentWithEvaluations/components/CommentModalOptimized.tsx b/apps/web/src/components/DocumentWithEvaluations/components/CommentModalOptimized.tsx index a99e7d8c8..f6f971c69 100644 --- a/apps/web/src/components/DocumentWithEvaluations/components/CommentModalOptimized.tsx +++ b/apps/web/src/components/DocumentWithEvaluations/components/CommentModalOptimized.tsx @@ -7,13 +7,12 @@ import { Button } from "@/components/ui/button"; import { commentToYaml } from "@/shared/utils/commentToYaml"; import { parseColoredText } from "@/shared/utils/ui/coloredText"; import { - CheckIcon, ChevronDownIcon, ChevronRightIcon, DocumentDuplicateIcon, - XMarkIcon, } from "@heroicons/react/24/outline"; import type { Comment } from "@roast/ai"; +import { CommentVariantIndicator } from "@/components/CommentVariantIndicator"; import { MARKDOWN_COMPONENTS, MARKDOWN_PLUGINS } from "../config/markdown"; interface CommentModalOptimizedProps { @@ -30,43 +29,6 @@ interface CommentModalOptimizedProps { onGetShareLink?: (commentId: string) => string; } -function getLevelIndicator(level: string) { - let bgColor = "bg-blue-500"; - let content: React.ReactNode; - - switch (level) { - case "error": - bgColor = "bg-red-500"; - content = ; - break; - case "warning": - bgColor = "bg-amber-500"; - content = ( - ! - ); - break; - case "success": - bgColor = "bg-green-500"; - content = ; - break; - case "info": - default: - bgColor = "bg-blue-500"; - content = ( - i - ); - break; - } - - return ( -
- {content} -
- ); -} - export const CommentModalOptimized = memo(function CommentModalOptimized({ comments, currentCommentId, @@ -178,7 +140,7 @@ export const CommentModalOptimized = memo(function CommentModalOptimized({ if (!currentData || !isOpen) return null; const { comment, agentName } = currentData; - const level = comment.level || "info"; + const variant = comment.variant || "info"; return (
- {getLevelIndicator(level)} +

{comment.header ? ( diff --git a/apps/web/src/components/DocumentWithEvaluations/components/EvaluationAnalysisSection.tsx b/apps/web/src/components/DocumentWithEvaluations/components/EvaluationAnalysisSection.tsx index 8394657a1..fde7aa430 100644 --- a/apps/web/src/components/DocumentWithEvaluations/components/EvaluationAnalysisSection.tsx +++ b/apps/web/src/components/DocumentWithEvaluations/components/EvaluationAnalysisSection.tsx @@ -129,7 +129,7 @@ export function EvaluationAnalysisSection({ evaluationVersionId: evaluation.id || "", highlightId: `${evaluation.agentId}-highlight-${index}`, header: comment.header ?? null, - level: comment.level ?? null, + variant: comment.variant ?? null, source: comment.source ?? null, metadata: comment.metadata ?? null, highlight: { diff --git a/apps/web/src/components/DocumentWithEvaluations/components/EvaluationView.tsx b/apps/web/src/components/DocumentWithEvaluations/components/EvaluationView.tsx index 4046da07b..1cba013e6 100644 --- a/apps/web/src/components/DocumentWithEvaluations/components/EvaluationView.tsx +++ b/apps/web/src/components/DocumentWithEvaluations/components/EvaluationView.tsx @@ -9,7 +9,7 @@ import { cn } from "@/lib/utils"; import type { Comment as DbComment } from "@/shared/types/databaseTypes"; import { dbCommentToAiComment } from "@/shared/utils/typeAdapters"; import { getValidAndSortedComments } from "@/shared/utils/ui/commentUtils"; -import type { Comment } from "@roast/ai"; +import type { Comment, CommentVariant } from "@roast/ai"; import { LAYOUT } from "../constants"; import { LocalCommentsUIProvider } from "../context/LocalCommentsUIContext"; @@ -22,10 +22,10 @@ import { EvaluationAnalysisSection } from "./EvaluationAnalysisSection"; import { EvaluationCardsHeader } from "./EvaluationCardsHeader"; /** - * Maps comment levels to appropriate highlight colors + * Maps comment variants to appropriate highlight colors */ -function getLevelHighlightColor(level?: string | null): string { - switch (level) { +function getVariantHighlightColor(variant?: CommentVariant | null): string { + switch (variant) { case "error": return "#dc2626"; // Brighter red - for false claims (more intense) case "warning": @@ -103,7 +103,7 @@ export function EvaluationView({ if (localShowDebugComments) { return allComments; } - return allComments.filter((comment) => comment.level !== "debug"); + return allComments.filter((comment) => comment.variant !== "debug"); }, [allComments, localShowDebugComments]) as Array< DbComment & { agentName: string } >; @@ -252,7 +252,7 @@ export function EvaluationView({ endOffset: comment.highlight.endOffset!, quotedText: comment.highlight.quotedText || "", tag: originalIndex.toString(), - color: getLevelHighlightColor(comment.level), + color: getVariantHighlightColor(comment.variant), }; }); }, [commentsWithHighlights, displayComments]); diff --git a/apps/web/src/components/DocumentWithEvaluations/components/PositionedComment.tsx b/apps/web/src/components/DocumentWithEvaluations/components/PositionedComment.tsx index c73416de5..14bb95141 100644 --- a/apps/web/src/components/DocumentWithEvaluations/components/PositionedComment.tsx +++ b/apps/web/src/components/DocumentWithEvaluations/components/PositionedComment.tsx @@ -8,15 +8,12 @@ import { AgentIcon } from "@/components/AgentIcon"; import { commentToYaml } from "@/shared/utils/commentToYaml"; import { parseColoredText } from "@/shared/utils/ui/coloredText"; import { - CheckIcon, ChevronDownIcon, ChevronRightIcon, DocumentDuplicateIcon, - XMarkIcon, } from "@heroicons/react/24/outline"; import type { Comment } from "@roast/ai"; - -import IndexFingerIcon from "../../../../public/app-icons/indexFinger.svg"; +import { CommentVariantIndicator } from "@/components/CommentVariantIndicator"; import { MARKDOWN_COMPONENTS, MARKDOWN_PLUGINS, @@ -87,49 +84,6 @@ export function PositionedComment({ // Note: We show header if available, otherwise description is shown inline - const level = comment.level || "info"; // Default to info if not set - // Get level indicator (icon with colored background) - const getLevelIndicator = (level: string, isHovered: boolean) => { - let bgColor = isHovered ? "bg-blue-500" : "bg-blue-400"; - let content: React.ReactNode; - - switch (level) { - case "error": - bgColor = isHovered ? "bg-red-500" : "bg-red-400"; - content = ; - break; - case "warning": - bgColor = isHovered ? "bg-orange-500" : "bg-orange-400"; - content = ( - ! - ); - break; - case "nitpick": - bgColor = isHovered ? "bg-fuchsia-500" : "bg-fuchsia-300"; - content = ; - break; - case "success": - bgColor = isHovered ? "bg-green-500" : "bg-green-300"; - content = ; - break; - case "info": - default: - bgColor = isHovered ? "bg-blue-500" : "bg-blue-400"; - content = ( - i - ); - break; - } - - return ( -
- {content} -
- ); - }; - return (
- {getLevelIndicator(level, isHovered)} + {parseColoredText(comment.header)}
{/* Agent name on the right */} diff --git a/apps/web/src/components/EvaluationComments.tsx b/apps/web/src/components/EvaluationComments.tsx index 85911258b..1bf4411b0 100644 --- a/apps/web/src/components/EvaluationComments.tsx +++ b/apps/web/src/components/EvaluationComments.tsx @@ -9,10 +9,10 @@ import { InformationCircleIcon, ExclamationTriangleIcon, XCircleIcon, - CheckCircleIcon, - XMarkIcon } from "@heroicons/react/24/outline"; import { parseColoredText } from "@/shared/utils/ui/coloredText"; +import { CommentVariantIndicatorCompact } from "@/components/CommentVariantIndicator"; +import type { CommentVariant } from "@roast/ai"; // Database comment type (different from the documentSchema Comment type) type DatabaseComment = { @@ -23,7 +23,7 @@ type DatabaseComment = { evaluationVersionId: string; highlightId: string; header?: string | null; - level?: string | null; + variant?: CommentVariant | null; source?: string | null; metadata?: Record | null; highlight: { @@ -42,37 +42,6 @@ interface EvaluationCommentsProps { documentContent?: string; } -function getLevelIndicator(level?: string | null) { - let bgColor = "bg-blue-400"; - let content: React.ReactNode; - - switch (level) { - case "error": - bgColor = "bg-red-500"; - content = ; - break; - case "warning": - bgColor = "bg-amber-500"; - content = !; - break; - case "success": - bgColor = "bg-green-500"; - content = ; - break; - case "info": - case "debug": - default: - bgColor = "bg-blue-500"; - content = i; - break; - } - - return ( -
- {content} -
- ); -} export function EvaluationComments({ comments, @@ -102,7 +71,7 @@ export function EvaluationComments({ id={`comment-${index + 1}`} className="scroll-mt-4 flex items-center gap-2 text-lg font-semibold text-gray-900" > - {getLevelIndicator(comment.level)} + {comment.header ? parseColoredText(comment.header) : `Comment ${index + 1}`} @@ -154,18 +123,19 @@ export function EvaluationComments({ {/* Comment content with light background */}
- {/* Level and source badges */} + {/* Variant and source badges */}
- {comment.level && ( + {comment.variant && ( - {comment.level} + {comment.variant} )} {comment.source && ( diff --git a/apps/web/src/infrastructure/export/document-export-service.ts b/apps/web/src/infrastructure/export/document-export-service.ts index a8a6edb76..88cd54c00 100644 --- a/apps/web/src/infrastructure/export/document-export-service.ts +++ b/apps/web/src/infrastructure/export/document-export-service.ts @@ -48,7 +48,7 @@ abstract class BaseDocumentExporter implements DocumentExporter { return comments.map((comment: Comment) => ({ header: comment.header, - level: comment.level, + variant: comment.variant, source: comment.source, metadata: comment.metadata, description: comment.description, @@ -128,7 +128,7 @@ class MarkdownExporter extends BaseDocumentExporter { sections.push(`${idx + 1}. **${header}**`); const badges = []; - if (comment.level) badges.push(`[${comment.level.toUpperCase()}]`); + if (comment.variant) badges.push(`[${comment.variant.toUpperCase()}]`); if (comment.source) badges.push(`[${comment.source}]`); if (badges.length > 0) { sections.push(` ${badges.join(" ")}`); diff --git a/apps/web/src/models/Document.ts b/apps/web/src/models/Document.ts index faddeb88e..1154a6284 100644 --- a/apps/web/src/models/Document.ts +++ b/apps/web/src/models/Document.ts @@ -155,7 +155,7 @@ class DocumentTransformer { error: comment.highlight.error, } : null, header: getCommentProperty(comment, 'header', null), - level: getCommentProperty(comment, 'level', null), + variant: getCommentProperty(comment, 'variant', null), source: getCommentProperty(comment, 'source', null), metadata: getCommentProperty(comment, 'metadata', null), }; diff --git a/apps/web/src/shared/types/commentTypes.ts b/apps/web/src/shared/types/commentTypes.ts index 984354a74..9d266716b 100644 --- a/apps/web/src/shared/types/commentTypes.ts +++ b/apps/web/src/shared/types/commentTypes.ts @@ -5,14 +5,14 @@ * and database comment structures. */ -import type { Comment as AIComment } from '@roast/ai'; +import type { Comment as AIComment, CommentVariant } from '@roast/ai'; import type { Comment as DatabaseComment } from './databaseTypes'; /** * Extended comment interface that includes all possible fields * from both AI and database representations */ -export interface ExtendedComment extends Partial> { +export interface ExtendedComment extends Partial> { // Required database fields id?: string; commentText?: string; @@ -21,7 +21,7 @@ export interface ExtendedComment extends Partial | null; @@ -82,7 +82,7 @@ export function aiCommentToDatabaseComment( highlight, // Add any extended fields that might be in the comment ...(comment.header !== undefined && { header: comment.header }), - ...(comment.level !== undefined && { level: comment.level }), + ...(comment.variant !== undefined && { variant: comment.variant }), ...(comment.source !== undefined && { source: comment.source }), ...(comment.metadata !== undefined && { metadata: comment.metadata }), } as DatabaseComment; diff --git a/apps/web/src/shared/types/validationSchemas.ts b/apps/web/src/shared/types/validationSchemas.ts index 3bdf326a8..d1da5ae8d 100644 --- a/apps/web/src/shared/types/validationSchemas.ts +++ b/apps/web/src/shared/types/validationSchemas.ts @@ -5,6 +5,8 @@ */ import { z } from "zod"; +// Zod schema for comment variant enum +export const CommentVariantSchema = z.enum(['error', 'warning', 'nitpick', 'info', 'success', 'debug']); // Schema for document highlight validation export const HighlightValidationSchema = z.object({ @@ -32,7 +34,7 @@ export const CommentValidationSchema = z.object({ // Plugin standardization fields header: z.string().optional(), // Concise summary like "2+2=5 ā†’ 2+2=4" - level: z.enum(['error', 'warning', 'info', 'success']).optional(), + variant: CommentVariantSchema.optional(), source: z.string().optional(), // Plugin identifier: 'math', 'spelling', etc. metadata: z.record(z.string(), z.any()).optional(), // Plugin-specific data }); diff --git a/apps/web/src/shared/utils/commentToYaml.ts b/apps/web/src/shared/utils/commentToYaml.ts index 9b0bbe705..9401f29b4 100644 --- a/apps/web/src/shared/utils/commentToYaml.ts +++ b/apps/web/src/shared/utils/commentToYaml.ts @@ -27,8 +27,8 @@ export function commentToYaml(comment: Comment, agentName: string): string { // Metadata yamlLines.push(` agent: "${agentName}"`); - if (comment.level) { - yamlLines.push(` level: ${comment.level}`); + if (comment.variant) { + yamlLines.push(` variant: ${comment.variant}`); } if (comment.source) { diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/builder.ts b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/builder.ts index 87329619c..98a785c8f 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/builder.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/builder.ts @@ -5,7 +5,7 @@ import type { VerifiedFact } from '../VerifiedFact'; import { buildDescription, buildTitle, - getLevel, + getVariant, buildObservation, buildSignificance, buildGrade @@ -31,7 +31,7 @@ export async function buildFactComment( // Get markdown content from pure functions const description = buildDescription(fact); const header = buildTitle(fact); - const level = getLevel(fact); + const variant = getVariant(fact); const observation = buildObservation(fact); const significance = buildSignificance(fact); const grade = buildGrade(fact); @@ -48,7 +48,7 @@ export async function buildFactComment( // Structured content header, - level, + variant, observation, significance, grade, diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/markdown.ts b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/markdown.ts index 73c6cb482..ee2a8ec3f 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/markdown.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/comments/markdown.ts @@ -1,6 +1,7 @@ import type { VerifiedFact } from '../VerifiedFact'; import { escapeXml } from '../../../../shared/utils/xml'; import { LIMITS, THRESHOLDS } from '../constants'; +import type { CommentVariant } from '@roast/ai'; /** * Pure functions for generating markdown content for fact-check comments. @@ -156,9 +157,9 @@ export function buildTitle(fact: VerifiedFact): string { } /** - * Get the severity level for a fact comment + * Get the visual variant for a fact comment */ -export function getLevel(fact: VerifiedFact): "error" | "warning" | "info" | "success" | "debug" { +export function getVariant(fact: VerifiedFact): CommentVariant { const verdict = fact.verification?.verdict; if (verdict === "false") return "error"; if (verdict === "partially-true") return "warning"; diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-e2e.integration.vtest.ts b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-e2e.integration.vtest.ts index 1b4970d96..031383d65 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-e2e.integration.vtest.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-e2e.integration.vtest.ts @@ -45,12 +45,12 @@ The speed of light in a vacuum is approximately 299,792 kilometers per second. // Check that comments have proper structure const firstComment = result.comments[0]; if (firstComment) { - expect(firstComment.plugin).toBe('fact-check'); - expect(firstComment.location).toBeDefined(); - expect(firstComment.location.startOffset).toBeGreaterThanOrEqual(0); - expect(firstComment.location.endOffset).toBeGreaterThan(firstComment.location.startOffset); + expect(firstComment.source).toBe('fact-check'); + expect(firstComment.highlight).toBeDefined(); + expect(firstComment.highlight.startOffset).toBeGreaterThanOrEqual(0); + expect(firstComment.highlight.endOffset).toBeGreaterThan(firstComment.highlight.startOffset); expect(firstComment.description).toBeDefined(); - expect(firstComment.level).toMatch(/^(error|warning|info|success|debug)$/); + expect(firstComment.variant).toMatch(/^(error|warning|info|success|debug)$/); } // Verify summary was generated @@ -92,7 +92,7 @@ The human body has 206 bones (adults actually have 206, this is correct). const result = await plugin.analyze(chunks, documentText); // Should identify false claims - expect(result.comments.some(c => c.level === 'error')).toBe(true); + expect(result.comments.some(c => c.variant === 'error')).toBe(true); // Analysis should mention false claims expect(result.analysis.toLowerCase()).toMatch(/false|incorrect|error/); @@ -136,17 +136,17 @@ The human body has 206 bones (adults actually have 206, this is correct). // If claims were found, they should have valid locations if (result.comments.length > 0) { result.comments.forEach(comment => { - if (comment.location) { + if (comment.highlight) { // Location should be within document bounds - expect(comment.location.startOffset).toBeGreaterThanOrEqual(0); - expect(comment.location.endOffset).toBeLessThanOrEqual(documentText.length); + expect(comment.highlight.startOffset).toBeGreaterThanOrEqual(0); + expect(comment.highlight.endOffset).toBeLessThanOrEqual(documentText.length); // Quoted text should match the document at that location const extractedText = documentText.substring( - comment.location.startOffset, - comment.location.endOffset + comment.highlight.startOffset, + comment.highlight.endOffset ); - expect(comment.location.quotedText).toBeTruthy(); + expect(comment.highlight.quotedText).toBeTruthy(); } }); } @@ -198,7 +198,7 @@ We have achieved 99.9% customer satisfaction across 10 million users. ); trivialClaims.forEach(claim => { - expect(claim.level).toMatch(/^(debug|info)$/); + expect(claim.variant).toMatch(/^(debug|info)$/); }); }, 60000); }); \ No newline at end of file diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-location.vtest.ts b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-location.vtest.ts index 1c0a175a9..b4a9464f3 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-location.vtest.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/fact-check-location.vtest.ts @@ -83,20 +83,20 @@ describe('FactCheckPlugin Location Tracking', () => { // Verify the location is document-absolute, not chunk-relative const comment = result.comments[0]; - if (comment && comment.location) { + if (comment && comment.highlight) { // The fact "The Earth orbits the Sun." starts at position 30 in the document // (12 for chunk start + 18 for position within chunk) const expectedStart = 12 + 18; // chunk start + relative offset const expectedEnd = 12 + 43; // chunk start + relative end offset - expect(comment.location.startOffset).toBe(expectedStart); - expect(comment.location.endOffset).toBe(expectedEnd); - expect(comment.location.quotedText).toBe('The Earth orbits the Sun.'); + expect(comment.highlight.startOffset).toBe(expectedStart); + expect(comment.highlight.endOffset).toBe(expectedEnd); + expect(comment.highlight.quotedText).toBe('The Earth orbits the Sun.'); // Verify the text at the location matches const extractedText = documentText.substring( - comment.location.startOffset, - comment.location.endOffset + comment.highlight.startOffset, + comment.highlight.endOffset ); expect(extractedText).toBe('The Earth orbits the Sun.'); } @@ -118,8 +118,8 @@ describe('FactCheckPlugin Location Tracking', () => { // Should still generate results but location will use fallback (0 offset) expect(result.comments).toBeDefined(); - if (result.comments.length > 0 && result.comments[0]?.location) { - const { startOffset, endOffset, quotedText } = result.comments[0].location; + if (result.comments.length > 0 && result.comments[0]?.highlight) { + const { startOffset, endOffset, quotedText } = result.comments[0].highlight; expect(startOffset).toBeGreaterThanOrEqual(0); expect(endOffset).toBeGreaterThan(startOffset); expect(documentText.substring(startOffset, endOffset)).toBe(quotedText); @@ -148,7 +148,7 @@ describe('FactCheckPlugin Location Tracking', () => { // Should not generate comments since location verification would fail expect(result.comments).toBeDefined(); // Comments should be filtered out if location can't be verified - const regularComments = result.comments.filter(c => c.level !== 'debug'); + const regularComments = result.comments.filter(c => c.variant !== 'debug'); expect(regularComments.length).toBe(0); }); }); \ No newline at end of file diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/index.ts b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/index.ts index 62634d4c6..4c5f1940f 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fact-check/index.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fact-check/index.ts @@ -466,7 +466,7 @@ export class FactCheckPlugin implements SimpleAnalysisPlugin { toolChain, header: `Fact claim location not found`, - level: "debug" as const, + variant: "debug" as const, description: `The fact-checker found this claim but couldn't locate it precisely in the document: "${fact.text}". This might be due to text paraphrasing or formatting differences between extraction and document structure.`, }); } diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/FallacyIssue.ts b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/FallacyIssue.ts index dbea08ab5..b9a5848e2 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/FallacyIssue.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/FallacyIssue.ts @@ -1,5 +1,5 @@ import { logger } from "../../../shared/logger"; -import type { DocumentLocation } from "../../../shared/types"; +import type { CommentVariant, DocumentLocation } from "../../../shared/types"; import type { ExtractedFallacyIssue } from "../../../tools/fallacy-extractor/types"; import fuzzyTextLocatorTool from "../../../tools/smart-text-searcher"; import type { ToolContext } from "../../../tools/base/Tool"; @@ -60,11 +60,11 @@ export class FallacyIssue { } /** - * Get the comment level for this issue + * Get the comment variant for this issue * * Uses importance score as primary factor, with severity for critical issues */ - getCommentLevel(): "error" | "warning" | "nitpick" | "info" | "success" | "debug" { + getCommentVariant(): CommentVariant { const { issueType, importanceScore, severityScore } = this.issue; // Verified accurate claims get success diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/builder.ts b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/builder.ts index 03686a736..2b2bd761c 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/builder.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/builder.ts @@ -5,7 +5,7 @@ import type { FallacyIssue } from "../FallacyIssue"; import { buildDescription, buildTitle, - getLevel, + getVariant, buildObservation, buildSignificance, getImportance, @@ -31,7 +31,7 @@ export async function buildFallacyComment( // Get markdown content from pure functions const description = buildDescription(issue); const header = buildTitle(issue); - const level = getLevel(issue); + const variant = getVariant(issue); const observation = buildObservation(issue); const significance = buildSignificance(issue); const importance = getImportance(issue); @@ -48,7 +48,7 @@ export async function buildFallacyComment( // Structured content header, - level, + variant, observation, significance, importance, diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/markdown.ts b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/markdown.ts index 186b4884c..549ed1886 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/markdown.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/comments/markdown.ts @@ -1,3 +1,4 @@ +import type { CommentVariant } from "../../../../shared/types"; import type { FallacyIssue } from "../FallacyIssue"; import { ISSUE_TYPES } from "../constants"; @@ -38,12 +39,10 @@ export function buildTitle(issue: FallacyIssue): string { } /** - * Get the severity level for an epistemic issue comment + * Get the visual variant for an epistemic issue comment */ -export function getLevel( - issue: FallacyIssue -): "error" | "warning" | "nitpick" | "info" | "success" | "debug" { - return issue.getCommentLevel(); +export function getVariant(issue: FallacyIssue): CommentVariant { + return issue.getCommentVariant(); } /** diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/fallacy-check-iteration.vtest.ts b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/fallacy-check-iteration.vtest.ts index a1edc09a8..127321d02 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/fallacy-check-iteration.vtest.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/fallacy-check-iteration.vtest.ts @@ -59,16 +59,16 @@ Traditional financial advisors tell you to "diversify" and "invest for the long output += `\nAnalysis:\n${result.analysis}\n`; output += `\nTotal comments: ${result.comments.length}\n`; - // Group by level + // Group by variant const byLevel = { - error: result.comments.filter(c => c.level === 'error'), - warning: result.comments.filter(c => c.level === 'warning'), - nitpick: result.comments.filter(c => c.level === 'nitpick'), - success: result.comments.filter(c => c.level === 'success'), - info: result.comments.filter(c => c.level === 'info'), + error: result.comments.filter(c => c.variant === 'error'), + warning: result.comments.filter(c => c.variant === 'warning'), + nitpick: result.comments.filter(c => c.variant === 'nitpick'), + success: result.comments.filter(c => c.variant === 'success'), + info: result.comments.filter(c => c.variant === 'info'), }; - output += `\nBy Level: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings, ${byLevel.nitpick.length} nitpicks, ${byLevel.success.length} success, ${byLevel.info.length} info\n`; + output += `\nBy Variant: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings, ${byLevel.nitpick.length} nitpicks, ${byLevel.success.length} success, ${byLevel.info.length} info\n`; // Analyze issue types const issueTypes = result.comments.map(c => { @@ -93,7 +93,7 @@ Traditional financial advisors tell you to "diversify" and "invest for the long output += ` - ${type}: ${count}\n`; }); - // Log each comment sorted by level + // Log each comment sorted by variant ['error', 'warning', 'success', 'info'].forEach(level => { const comments = byLevel[level as keyof typeof byLevel]; if (comments.length > 0) { @@ -101,8 +101,8 @@ Traditional financial advisors tell you to "diversify" and "invest for the long comments.forEach((comment, i) => { output += `\n--- ${level.toUpperCase()} ${i + 1} ---\n`; output += `Header: ${comment.header}\n`; - if (comment.location?.quotedText) { - output += `Location: ${comment.location.quotedText.substring(0, 150)}\n`; + if (comment.highlight?.quotedText) { + output += `Location: ${comment.highlight.quotedText.substring(0, 150)}\n`; } output += `\nDescription:\n${comment.description}\n`; if (comment.importance) { diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/index.ts b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/index.ts index 14a46db7a..ddc5ef848 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/index.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/index.ts @@ -207,7 +207,7 @@ export class FallacyCheckPlugin implements SimpleAnalysisPlugin { index, header: comment.header || "Epistemic Issue", description: comment.description, - level: comment.level || 'warning', + variant: comment.variant || 'warning', importance: comment.importance, quotedText: comment.highlight.quotedText, })); diff --git a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/realistic-test.llm.vtest.ts b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/realistic-test.llm.vtest.ts index ea11606e6..58a7129f8 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/realistic-test.llm.vtest.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/fallacy-check/realistic-test.llm.vtest.ts @@ -149,23 +149,23 @@ Traditional business advice tells you to spend years planning. We spent 2 months console.log(`\n${result.analysis}`); console.log(`\nšŸ’¬ Total Comments: ${result.comments.length}\n`); - // Group by level - const byLevel = { - error: result.comments.filter(c => c.level === 'error'), - warning: result.comments.filter(c => c.level === 'warning'), - nitpick: result.comments.filter(c => c.level === 'nitpick'), - success: result.comments.filter(c => c.level === 'success'), - info: result.comments.filter(c => c.level === 'info'), + // Group by variant + const byVariant = { + error: result.comments.filter(c => c.variant === 'error'), + warning: result.comments.filter(c => c.variant === 'warning'), + nitpick: result.comments.filter(c => c.variant === 'nitpick'), + success: result.comments.filter(c => c.variant === 'success'), + info: result.comments.filter(c => c.variant === 'info'), }; - console.log(`By Level: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings, ${byLevel.nitpick.length} nitpicks, ${byLevel.success.length} success, ${byLevel.info.length} info\n`); + console.log(`By Variant: ${byVariant.error.length} errors, ${byVariant.warning.length} warnings, ${byVariant.nitpick.length} nitpicks, ${byVariant.success.length} success, ${byVariant.info.length} info\n`); // Show first 3 issues console.log('Top 3 Issues:'); result.comments.slice(0, 3).forEach((comment, i) => { - console.log(`\n${i + 1}. [${comment.level.toUpperCase()}] ${comment.header}`); - if (comment.location?.quotedText) { - console.log(` Text: "${comment.location.quotedText.substring(0, 80)}..."`); + console.log(`\n${i + 1}. [${comment.variant?.toUpperCase()}] ${comment.header}`); + if (comment.highlight?.quotedText) { + console.log(` Text: "${comment.highlight.quotedText.substring(0, 80)}..."`); } console.log(` ${comment.description.substring(0, 200)}...`); }); @@ -173,7 +173,7 @@ Traditional business advice tells you to spend years planning. We spent 2 months // Should catch: survivorship bias, cherry-picked 2020, false precision (847.3%), // vague claims, selection bias, false dichotomy, appeal to emotion expect(result.comments.length).toBeGreaterThan(5); - expect(byLevel.error.length).toBeGreaterThan(3); + expect(byVariant.error.length).toBeGreaterThan(3); }, 300000); // 5 minutes for LLM tests @@ -192,16 +192,16 @@ Traditional business advice tells you to spend years planning. We spent 2 months console.log(`\nšŸ’¬ Total Comments: ${result.comments.length}\n`); const byLevel = { - error: result.comments.filter(c => c.level === 'error'), - warning: result.comments.filter(c => c.level === 'warning'), + error: result.comments.filter(c => c.variant === 'error'), + warning: result.comments.filter(c => c.variant === 'warning'), }; - console.log(`By Level: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings\n`); + console.log(`By Variant: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings\n`); // Show first 3 issues console.log('Top 3 Issues:'); result.comments.slice(0, 3).forEach((comment, i) => { - console.log(`\n${i + 1}. [${comment.level.toUpperCase()}] ${comment.header}`); + console.log(`\n${i + 1}. [${comment.variant?.toUpperCase()}] ${comment.header}`); console.log(` ${comment.description.substring(0, 200)}...`); }); @@ -225,17 +225,17 @@ Traditional business advice tells you to spend years planning. We spent 2 months console.log(`\n${result.analysis}`); console.log(`\nšŸ’¬ Total Comments: ${result.comments.length}\n`); - const byLevel = { - error: result.comments.filter(c => c.level === 'error'), - warning: result.comments.filter(c => c.level === 'warning'), + const byVariant = { + error: result.comments.filter(c => c.variant === 'error'), + warning: result.comments.filter(c => c.variant === 'warning'), }; - console.log(`By Level: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings\n`); + console.log(`By Variant: ${byVariant.error.length} errors, ${byVariant.warning.length} warnings\n`); // Show all issues for political piece console.log('All Issues:'); result.comments.forEach((comment, i) => { - console.log(`\n${i + 1}. [${comment.level.toUpperCase()}] ${comment.header}`); + console.log(`\n${i + 1}. [${comment.variant?.toUpperCase()}] ${comment.header}`); console.log(` ${comment.description.substring(0, 150)}...`); }); @@ -260,16 +260,16 @@ Traditional business advice tells you to spend years planning. We spent 2 months console.log(`\nšŸ’¬ Total Comments: ${result.comments.length}\n`); const byLevel = { - error: result.comments.filter(c => c.level === 'error'), - warning: result.comments.filter(c => c.level === 'warning'), + error: result.comments.filter(c => c.variant === 'error'), + warning: result.comments.filter(c => c.variant === 'warning'), }; - console.log(`By Level: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings\n`); + console.log(`By Variant: ${byLevel.error.length} errors, ${byLevel.warning.length} warnings\n`); // Show first 3 issues console.log('Top 3 Issues:'); result.comments.slice(0, 3).forEach((comment, i) => { - console.log(`\n${i + 1}. [${comment.level.toUpperCase()}] ${comment.header}`); + console.log(`\n${i + 1}. [${comment.variant?.toUpperCase()}] ${comment.header}`); console.log(` ${comment.description.substring(0, 200)}...`); }); diff --git a/internal-packages/ai/src/analysis-plugins/plugins/forecast/index.ts b/internal-packages/ai/src/analysis-plugins/plugins/forecast/index.ts index 1fdfc89e9..c3367d7fc 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/forecast/index.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/forecast/index.ts @@ -1,6 +1,7 @@ import { logger } from "../../../shared/logger"; import type { Comment, + CommentVariant, ToolChainResult, } from "../../../shared/types"; import type { @@ -180,7 +181,7 @@ class ExtractedForecast { return `šŸ“Š Forecast: ${truncated}`; } - private getLevel(): "error" | "warning" | "info" | "success" | "debug" { + private getVariant(): CommentVariant { // If we have both author and our forecast, base level on confidence gap if (this.ourForecast && this.extractedForecast.authorProbability !== undefined) { const gap = Math.abs( @@ -259,7 +260,7 @@ class ExtractedForecast { // Required fields description, header: this.buildTitle(), - level: this.getLevel(), + variant: this.getVariant(), // Additional structured content observation: this.buildObservation(), @@ -702,7 +703,7 @@ export class ForecastPlugin implements SimpleAnalysisPlugin { toolChain, header: `Prediction Detected, Location Unknown`, - level: "debug" as const, + variant: "debug" as const, description: `**Prediction Found:** > "${forecast.extractedForecast.originalText}" diff --git a/internal-packages/ai/src/analysis-plugins/plugins/math/index.ts b/internal-packages/ai/src/analysis-plugins/plugins/math/index.ts index dd02b7527..b8fc514a0 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/math/index.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/math/index.ts @@ -136,7 +136,7 @@ export class HybridMathErrorWrapper { // Structured content header: this.buildHeader(), - level: this.getLevel(), + variant: this.getLevel(), observation: this.buildObservation(), significance: this.buildSignificance(), }); @@ -375,7 +375,7 @@ export class ExtractedMathExpression { (this.expression.hasError ? `Math Error: ${this.expression.originalText}` : `Math: ${this.expression.originalText}`), - level: this.expression.hasError + variant: this.expression.hasError ? ("error" as const) : this.expression.verificationStatus === "verified" ? ("success" as const) @@ -757,7 +757,7 @@ export class MathPlugin implements SimpleAnalysisPlugin { toolChain, header: `Math Expression Detected, Location Unknown`, - level: "debug" as const, + variant: "debug" as const, description: `**Expression Found:** > "${extractedExpression.expression.originalText}" @@ -813,7 +813,7 @@ The analysis may still be valid, but the highlighting won't be precise.`, toolChain, header: `Math Expression Detected, Unverifiable`, - level: "debug" as const, + variant: "debug" as const, description: `**Expression Found:** > "${hybridWrapper.expression.originalText}" @@ -868,7 +868,7 @@ This expression was detected but couldn't be verified automatically. This could toolChain, header: `Math Expression Detected, Skipped`, - level: "debug" as const, + variant: "debug" as const, description: `**Expression Found:** > "${extractedExpression.expression.originalText}" diff --git a/internal-packages/ai/src/analysis-plugins/plugins/math/math.vtest.ts b/internal-packages/ai/src/analysis-plugins/plugins/math/math.vtest.ts index 30e8f06e2..18fdb33a5 100644 --- a/internal-packages/ai/src/analysis-plugins/plugins/math/math.vtest.ts +++ b/internal-packages/ai/src/analysis-plugins/plugins/math/math.vtest.ts @@ -158,7 +158,7 @@ describe('MathPlugin', () => { expect(result.analysis.length).toBeGreaterThan(0); expect(result.comments).toHaveLength(2); // Now generates comments for both verified_true and verified_false // Check that the error comment does NOT contain duplicate header text - const errorComment = result.comments.find(c => c.level === 'error'); + const errorComment = result.comments.find(c => c.variant === 'error'); expect(errorComment).toBeDefined(); expect(errorComment?.description).not.toContain('[Math]'); expect(errorComment?.description).not.toContain('`; })(), - level: error.type === "grammar" ? "warning" : "error", + variant: error.type === "grammar" ? "warning" : "error", // Minimal description - required by CommentBuilder but not shown when header exists description: error.description || " ", significance: diff --git a/internal-packages/ai/src/analysis-plugins/utils/CommentBuilder.ts b/internal-packages/ai/src/analysis-plugins/utils/CommentBuilder.ts index 351afb71c..59c32efec 100644 --- a/internal-packages/ai/src/analysis-plugins/utils/CommentBuilder.ts +++ b/internal-packages/ai/src/analysis-plugins/utils/CommentBuilder.ts @@ -1,4 +1,4 @@ -import type { Comment, DocumentLocation, ToolChainResult, CommentMetadata } from '../../shared/types'; +import type { Comment, CommentVariant, DocumentLocation, ToolChainResult, CommentMetadata } from '../../shared/types'; import type { MathExtractResult, ForecastExtractResult, @@ -28,7 +28,7 @@ export interface CommentBuildOptions { // Optional overrides (auto-generated from tool chain if not provided) header?: string; description?: string; - level?: 'error' | 'warning' | 'nitpick' | 'info' | 'success' | 'debug'; + variant?: CommentVariant; importance?: number; // Optional structured fields @@ -62,13 +62,13 @@ export class CommentBuilder { // Plugins must provide all required fields if (!options.description) throw new Error('Description is required'); if (!options.header) throw new Error('Header is required'); - if (!options.level) throw new Error('Level is required'); + if (!options.variant) throw new Error('Variant is required'); return { // Required fields (must be provided by plugins) description: options.description, header: options.header, - level: options.level, + variant: options.variant, source: options.plugin, // Optional fields diff --git a/internal-packages/ai/src/index.ts b/internal-packages/ai/src/index.ts index b4efb0b66..ff5430e16 100644 --- a/internal-packages/ai/src/index.ts +++ b/internal-packages/ai/src/index.ts @@ -121,6 +121,7 @@ export * from './shared/utils/xml'; export type { Comment, CommentMetadata, + CommentVariant, ToolChainResult, DocumentLocation, LanguageConvention, diff --git a/internal-packages/ai/src/shared/types.ts b/internal-packages/ai/src/shared/types.ts index 509edfeed..35e435184 100644 --- a/internal-packages/ai/src/shared/types.ts +++ b/internal-packages/ai/src/shared/types.ts @@ -6,6 +6,11 @@ import type { ToolResult } from '../types/toolResults'; +/** + * Visual variant types for comment display + */ +export type CommentVariant = 'error' | 'warning' | 'nitpick' | 'info' | 'success' | 'debug'; + /** * Tool result in the analysis chain */ @@ -49,7 +54,7 @@ export interface Comment { // Optional fields for plugin-based comments header?: string; // Brief summary (provided by plugins) - level?: 'error' | 'warning' | 'nitpick' | 'info' | 'success' | 'debug'; // Severity level (provided by plugins) + variant?: CommentVariant; // Visual display variant (provided by plugins) source?: string; // Plugin name (provided by plugins) // Optional fields diff --git a/internal-packages/ai/src/tools/fallacy-review/index.ts b/internal-packages/ai/src/tools/fallacy-review/index.ts index 80a758397..143ba37bb 100644 --- a/internal-packages/ai/src/tools/fallacy-review/index.ts +++ b/internal-packages/ai/src/tools/fallacy-review/index.ts @@ -22,7 +22,7 @@ const reviewCommentSchema = z.object({ index: z.number(), header: z.string(), description: z.string(), - level: z.enum(['error', 'warning', 'nitpick', 'info', 'success', 'debug']), + variant: z.enum(['error', 'warning', 'nitpick', 'info', 'success', 'debug']), importance: z.number().optional(), quotedText: z.string(), }) satisfies z.ZodType; @@ -66,7 +66,7 @@ export class FallacyReviewTool extends Tool< .map((comment, idx) => { return `**Comment ${idx}**: Header: ${comment.header} -Level: ${comment.level} +Variant: ${comment.variant} Importance: ${comment.importance || 'N/A'} Quoted Text: "${comment.quotedText.substring(0, 100)}${comment.quotedText.length > 100 ? '...' : ''}" Description: ${comment.description} diff --git a/internal-packages/ai/src/tools/fallacy-review/types.ts b/internal-packages/ai/src/tools/fallacy-review/types.ts index ce3261dc3..3b51926d9 100644 --- a/internal-packages/ai/src/tools/fallacy-review/types.ts +++ b/internal-packages/ai/src/tools/fallacy-review/types.ts @@ -2,6 +2,8 @@ * Types for the fallacy review tool */ +import type { CommentVariant } from '@roast/ai'; + /** * Simplified comment representation for review */ @@ -15,8 +17,8 @@ export interface ReviewComment { /** Full description */ description: string; - /** Severity level */ - level: 'error' | 'warning' | 'nitpick' | 'info' | 'success' | 'debug'; + /** Visual variant for display */ + variant: CommentVariant; /** Importance score */ importance?: number; diff --git a/internal-packages/ai/src/tools/generated-schemas.ts b/internal-packages/ai/src/tools/generated-schemas.ts index f8c77aada..60b886b29 100644 --- a/internal-packages/ai/src/tools/generated-schemas.ts +++ b/internal-packages/ai/src/tools/generated-schemas.ts @@ -3,7 +3,7 @@ * Generated by scripts/generate-tool-schemas.ts * DO NOT EDIT MANUALLY * - * Schema Hash: 03fd6c8c36ea8ad38a61ccf1fc2e8ccca8017fecbcacc7106fea3f470eebbe59 + * Schema Hash: 149ba51c2993ee8603d5b4b51db2131071e5c8772d46fb660173ceb7889ed78b */ export const toolSchemas = { @@ -2295,7 +2295,7 @@ export const toolSchemas = { "description": { "type": "string" }, - "level": { + "variant": { "type": "string", "enum": [ "error", @@ -2317,7 +2317,7 @@ export const toolSchemas = { "index", "header", "description", - "level", + "variant", "quotedText" ], "additionalProperties": false diff --git a/internal-packages/ai/src/tools/link-validator/linkHighlightGenerator.ts b/internal-packages/ai/src/tools/link-validator/linkHighlightGenerator.ts index 8ddbb26cf..91447b5a6 100644 --- a/internal-packages/ai/src/tools/link-validator/linkHighlightGenerator.ts +++ b/internal-packages/ai/src/tools/link-validator/linkHighlightGenerator.ts @@ -187,7 +187,7 @@ export function generateLinkHighlights( // Required fields for Comment interface header: extractedUrl.url.length > 50 ? extractedUrl.url.substring(0, 47) + '...' : extractedUrl.url, - level: grade > 70 ? 'success' : grade > 30 ? 'warning' : 'error', + variant: grade > 70 ? 'success' : grade > 30 ? 'warning' : 'error', source: 'link-analysis', metadata: { pluginName: 'link-analysis', diff --git a/internal-packages/db/prisma/migrations/20251121132349_rename_level_to_variant/migration.sql b/internal-packages/db/prisma/migrations/20251121132349_rename_level_to_variant/migration.sql new file mode 100644 index 000000000..53ae7a3b0 --- /dev/null +++ b/internal-packages/db/prisma/migrations/20251121132349_rename_level_to_variant/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "EvaluationComment" RENAME COLUMN "level" TO "variant"; + diff --git a/internal-packages/db/prisma/schema.prisma b/internal-packages/db/prisma/schema.prisma index 1a7e5b8e7..58d4dbf5e 100644 --- a/internal-packages/db/prisma/schema.prisma +++ b/internal-packages/db/prisma/schema.prisma @@ -162,7 +162,7 @@ model EvaluationComment { evaluationVersionId String highlightId String @unique header String? - level String? + variant String? metadata Json? source String? evaluationVersion EvaluationVersion @relation(fields: [evaluationVersionId], references: [id], onDelete: Cascade) diff --git a/internal-packages/jobs/src/core/JobOrchestrator.ts b/internal-packages/jobs/src/core/JobOrchestrator.ts index 2a5815302..93e3ee86e 100644 --- a/internal-packages/jobs/src/core/JobOrchestrator.ts +++ b/internal-packages/jobs/src/core/JobOrchestrator.ts @@ -383,7 +383,7 @@ export class JobOrchestrator implements JobOrchestratorInterface { importance: comment.importance || null, grade: comment.grade || null, header: comment.header || null, - level: comment.level || null, + variant: comment.variant || null, source: comment.source || null, metadata: comment.metadata || null, evaluationVersionId,