Complete regex parser replacement with lexer/parser architecture

internal/archive/spec_merger.go

@@ -4,9 +4,9 @@ package archive
 import (
 	"fmt"
 	"os"
-	"regexp"
 	"strings"
 
+	"github.com/connerohnesorge/spectr/internal/mdparser"
 	"github.com/connerohnesorge/spectr/internal/parsers"
 )
 
@@ -182,110 +182,311 @@ func reconstructSpec(
 	reqMap map[string]parsers.RequirementBlock,
 	added []parsers.RequirementBlock,
 ) string {
-	// Split spec into: preamble, requirements section, after
-	preamble, reqsContent, after := splitSpec(baseContent)
+	// Parse the base content into AST
+	doc, err := mdparser.Parse(baseContent)
+	if err != nil {
+		// Fallback to empty doc on parse error
+		doc = &mdparser.Document{Children: []mdparser.Node{}}
+	}
 
-	// Extract original requirement order from base content
-	orderedReqs := extractOrderedRequirements(reqsContent, reqMap)
+	// Find Requirements section index and extract ordering
+	reqsSectionIdx, orderedReqs := extractRequirementsSection(doc, reqMap)
+	missingRequirementsSection := reqsSectionIdx == -1
+	if missingRequirementsSection {
+		orderedReqs = collectRequirementsWithoutSection(doc, reqMap)
+	}
 
-	// Build requirements section
-	var reqsBuilder strings.Builder
-	for i := range orderedReqs {
-		if i > 0 {
-			reqsBuilder.WriteString(newlineChar)
+	reqStartOffset := len(baseContent)
+	headerLevel := 2
+	headerText := "Requirements"
+	afterReqsIdx := -1
+	if reqsSectionIdx >= 0 && reqsSectionIdx < len(doc.Children) {
+		reqStartOffset = doc.Children[reqsSectionIdx].Pos().Offset
+
+		if header, ok := doc.Children[reqsSectionIdx].(*mdparser.Header); ok {
+			headerLevel = header.Level
+			headerText = header.Text
 		}
-		reqsBuilder.WriteString(
-			strings.TrimRight(orderedReqs[i].Raw, newlineChar),
-		)
-		reqsBuilder.WriteString(newlineChar)
+
+		afterReqsIdx = findNextSectionAfterRequirements(doc, reqsSectionIdx)
+	} else {
+		firstReqIdx := findFirstRequirementHeaderIndex(doc)
+		if firstReqIdx >= 0 && firstReqIdx < len(doc.Children) {
+			reqStartOffset = doc.Children[firstReqIdx].Pos().Offset
+			afterReqsIdx = findNextSectionAfterRequirements(doc, firstReqIdx)
+		}
+	}
+	reqEndOffset := len(baseContent)
+	if afterReqsIdx >= 0 && afterReqsIdx < len(doc.Children) {
+		reqEndOffset = doc.Children[afterReqsIdx].Pos().Offset
 	}
 
-	// Add new requirements at the end
-	for _, req := range added {
-		reqsBuilder.WriteString(newlineChar)
-		reqsBuilder.WriteString(strings.TrimRight(req.Raw, newlineChar))
-		reqsBuilder.WriteString(newlineChar)
+	preamble := baseContent[:reqStartOffset]
+	epilogue := ""
+	if reqEndOffset >= 0 && reqEndOffset <= len(baseContent) {
+		epilogue = baseContent[reqEndOffset:]
 	}
 
-	// Combine all parts
+	// Rebuild document with updated requirements
 	var result strings.Builder
 	result.WriteString(preamble)
-	result.WriteString(reqsBuilder.String())
-	result.WriteString(after)
 
-	// Normalize blank lines (collapse 3+ newlines to 2)
-	output := result.String()
-	multiNewline := regexp.MustCompile(`\n{3,}`)
-	output = multiNewline.ReplaceAllString(output, "\n\n")
+	if missingRequirementsSection && len(strings.TrimSpace(preamble)) > 0 {
+		if !strings.HasSuffix(result.String(), "\n") {
+			result.WriteString("\n")
+		}
+		if !strings.HasSuffix(result.String(), "\n\n") {
+			result.WriteString("\n")
+		}
+	}
+
+	result.WriteString(strings.Repeat("#", headerLevel))
+	result.WriteString(" ")
+	result.WriteString(headerText)
+	result.WriteString("\n\n")
+
+	// Write ordered requirements
+	for i, req := range orderedReqs {
+		if i > 0 {
+			result.WriteString("\n")
+		}
+		result.WriteString(strings.TrimRight(req.Raw, newlineChar))
+		result.WriteString("\n")
+	}
+
+	// Add new requirements at the end
+	for _, req := range added {
+		result.WriteString("\n")
+		result.WriteString(strings.TrimRight(req.Raw, newlineChar))
+		result.WriteString("\n")
+	}
 
-	return output
+	if epilogue != "" {
+		if !strings.HasSuffix(result.String(), "\n") {
+			result.WriteString("\n")
+		}
+		if !strings.HasSuffix(result.String(), "\n\n") {
+			result.WriteString("\n")
+		}
+		result.WriteString(epilogue)
+	}
+
+	// Normalize blank lines while preserving original formatting
+	return normalizeBlankLines(result.String())
 }
 
-// splitSpec splits spec into preamble, requirements section content, and after
-func splitSpec(content string) (preamble, requirements, after string) {
-	// Find ## Requirements header
-	reqHeaderPattern := regexp.MustCompile(`(?m)^##\s+Requirements\s*$`)
-	match := reqHeaderPattern.FindStringIndex(content)
-	if match == nil {
-		// No requirements section, return everything as preamble
-		return content, "", ""
+// extractRequirementsSection finds the Requirements section and extracts requirements in order
+// Returns: (sectionIndex, orderedRequirements)
+func extractRequirementsSection(
+	doc *mdparser.Document,
+	reqMap map[string]parsers.RequirementBlock,
+) (int, []parsers.RequirementBlock) {
+	// Find Requirements section header (H2)
+	reqsSectionIdx := -1
+	for i, node := range doc.Children {
+		header, ok := node.(*mdparser.Header)
+		if !ok || header.Level != 2 {
+			continue
+		}
+		if strings.TrimSpace(header.Text) == "Requirements" {
+			reqsSectionIdx = i
+
+			break
+		}
 	}
 
-	preamble = content[:match[1]] + "\n\n"
+	if reqsSectionIdx == -1 {
+		// No Requirements section found
+		return -1, nil
+	}
 
-	// Find next ## header after Requirements
-	nextHeaderPattern := regexp.MustCompile(`(?m)^##\s+`)
-	remainingContent := content[match[1]:]
-	nextMatch := nextHeaderPattern.FindStringIndex(remainingContent)
+	// Find the end of the Requirements section (next H2 or end of document)
+	endIdx := len(doc.Children)
+	for i := reqsSectionIdx + 1; i < len(doc.Children); i++ {
+		if header, ok := doc.Children[i].(*mdparser.Header); ok && header.Level == 2 {
+			endIdx = i
 
-	if nextMatch != nil {
-		requirements = remainingContent[:nextMatch[0]]
-		after = remainingContent[nextMatch[0]:]
-	} else {
-		requirements = remainingContent
-		after = ""
+			break
+		}
+	}
+
+	// Extract requirements in order from the section
+	var ordered []parsers.RequirementBlock
+	for i := reqsSectionIdx + 1; i < endIdx; i++ {
+		header, ok := doc.Children[i].(*mdparser.Header)
+		if !ok || header.Level != 3 {
+			continue
+		}
+
+		// Check if this is a requirement header
+		if !strings.HasPrefix(header.Text, "Requirement: ") {
+			continue
+		}
+
+		// Extract requirement name
+		name := strings.TrimPrefix(header.Text, "Requirement: ")
+		name = strings.TrimSpace(name)
+		normalized := parsers.NormalizeRequirementName(name)
+
+		// Look up in reqMap
+		req, exists := reqMap[normalized]
+		if !exists {
+			continue
+		}
+
+		ordered = append(ordered, req)
+		// Remove from map so we don't add duplicates
+		delete(reqMap, normalized)
 	}
 
-	return preamble, requirements, after
+	// Add any remaining requirements from map (shouldn't happen in normal flow)
+	for _, req := range reqMap {
+		ordered = append(ordered, req)
+	}
+
+	return reqsSectionIdx, ordered
 }
 
-// extractOrderedRequirements preserves requirement ordering
-// from original content
-func extractOrderedRequirements(
-	reqsContent string,
+// collectRequirementsWithoutSection orders requirements using document order when the
+// Requirements section is missing, allowing us to still merge updated requirement blocks.
+func collectRequirementsWithoutSection(
+	doc *mdparser.Document,
 	reqMap map[string]parsers.RequirementBlock,
 ) []parsers.RequirementBlock {
 	var ordered []parsers.RequirementBlock
 
-	// Find requirement headers in order
-	reqPattern := regexp.MustCompile(`(?m)^###\s+Requirement:\s*(.+)$`)
-	matches := reqPattern.FindAllStringSubmatch(reqsContent, -1)
-
-	for _, match := range matches {
-		if len(match) <= 1 {
+	for _, node := range doc.Children {
+		header, ok := node.(*mdparser.Header)
+		if !ok || header.Level != 3 {
+			continue
+		}
+		if !strings.HasPrefix(header.Text, "Requirement: ") {
 			continue
 		}
 
-		name := strings.TrimSpace(match[1])
+		name := strings.TrimSpace(strings.TrimPrefix(header.Text, "Requirement: "))
 		normalized := parsers.NormalizeRequirementName(name)
+
 		req, exists := reqMap[normalized]
 		if !exists {
 			continue
 		}
 
 		ordered = append(ordered, req)
-		// Remove from map so we don't add duplicates
 		delete(reqMap, normalized)
 	}
 
-	// Add any remaining requirements from map (shouldn't happen in normal flow)
 	for _, req := range reqMap {
 		ordered = append(ordered, req)
 	}
 
 	return ordered
 }
 
+// findNextSectionAfterRequirements finds the index of the next H2 section after Requirements
+func findNextSectionAfterRequirements(doc *mdparser.Document, reqsSectionIdx int) int {
+	if reqsSectionIdx < 0 {
+		return -1
+	}
+
+	for i := reqsSectionIdx + 1; i < len(doc.Children); i++ {
+		if header, ok := doc.Children[i].(*mdparser.Header); ok && header.Level == 2 {
+			return i
+		}
+	}
+
+	return -1
+}
+
+// findFirstRequirementHeaderIndex returns the index of the first H3 requirement header in the document.
+func findFirstRequirementHeaderIndex(doc *mdparser.Document) int {
+	for i, node := range doc.Children {
+		header, ok := node.(*mdparser.Header)
+		if !ok || header.Level != 3 {
+			continue
+		}
+
+		if strings.HasPrefix(header.Text, "Requirement: ") {
+			return i
+		}
+	}
+
+	return -1
+}
+
+// renderNode converts an AST node back to markdown text
+func renderNode(sb *strings.Builder, node mdparser.Node) {
+	switch n := node.(type) {
+	case *mdparser.Header:
+		sb.WriteString(strings.Repeat("#", n.Level))
+		sb.WriteString(" ")
+		sb.WriteString(n.Text)
+		sb.WriteString("\n")
+
+	case *mdparser.Paragraph:
+		for _, line := range n.Lines {
+			sb.WriteString(line)
+			sb.WriteString("\n")
+		}
+
+	case *mdparser.CodeBlock:
+		sb.WriteString("```")
+		sb.WriteString(n.Language)
+		sb.WriteString("\n")
+		for _, line := range n.Lines {
+			sb.WriteString(line)
+			sb.WriteString("\n")
+		}
+		sb.WriteString("```\n")
+
+	case *mdparser.List:
+		for _, item := range n.Items {
+			marker := "- "
+			if n.Ordered {
+				marker = "1. "
+			}
+			if item != nil && item.Marker != "" {
+				marker = item.Marker
+			}
+			sb.WriteString(marker)
+			sb.WriteString(item.Text)
+			sb.WriteString("\n")
+		}
+
+	case *mdparser.BlankLine:
+		// Render blank lines but cap at 2 consecutive
+		count := n.Count
+		if count > 2 {
+			count = 2
+		}
+
+		for range count {
+			sb.WriteString("\n")
+		}
+	}
+}
+
+// normalizeBlankLines collapses 3+ consecutive newlines to 2
+func normalizeBlankLines(content string) string {
+	lines := strings.Split(content, "\n")
+	var result []string
+	blankCount := 0
+
+	for _, line := range lines {
+		if strings.TrimSpace(line) == "" {
+			blankCount++
+			if blankCount <= 2 {
+				result = append(result, line)
+			}
+		} else {
+			blankCount = 0
+			result = append(result, line)
+		}
+	}
+
+	return strings.Join(result, "\n")
+}
+
 // generateSpecSkeleton creates a new spec skeleton for a capability
 func generateSpecSkeleton(targetPath string) string {
 	// Extract capability name from path