Implement Typesense search integration for Azure Container Apps deployment

EssentialCSharp.Web/Controllers/SearchController.cs

@@ -0,0 +1,115 @@
+using EssentialCSharp.Web.Models;
+using EssentialCSharp.Web.Services;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.RateLimiting;
+
+namespace EssentialCSharp.Web.Controllers;
+
+[ApiController]
+[Route("api/[controller]")]
+[EnableRateLimiting("SearchEndpoint")]
+public class SearchController : ControllerBase
+{
+    private readonly ITypesenseSearchService _searchService;
+    private readonly ILogger<SearchController> _logger;
+
+    public SearchController(ITypesenseSearchService searchService, ILogger<SearchController> logger)
+    {
+        _searchService = searchService;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Search for content using Typesense
+    /// </summary>
+    /// <param name="request">The search request parameters</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Search results</returns>
+    [HttpPost]
+    public async Task<IActionResult> Search([FromBody] SearchRequest request, CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(request.Query))
+        {
+            return BadRequest(new { error = "Search query cannot be empty." });
+        }
+
+        if (request.Query.Length > 500)
+        {
+            return BadRequest(new { error = "Search query is too long. Maximum 500 characters." });
+        }
+
+        try
+        {
+            var result = await _searchService.SearchAsync(
+                request.Query, 
+                request.Page, 
+                Math.Min(request.PerPage, 50), // Limit max results per page
+                cancellationToken);
+
+            return Ok(result);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Search failed for query: {Query}", request.Query);
+            return StatusCode(500, new { error = "Search service temporarily unavailable." });
+        }
+    }
+
+    /// <summary>
+    /// Search for content using GET method for simple queries
+    /// </summary>
+    /// <param name="q">Search query</param>
+    /// <param name="page">Page number (default: 1)</param>
+    /// <param name="per_page">Results per page (default: 10, max: 50)</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Search results</returns>
+    [HttpGet]
+    public async Task<IActionResult> Search(
+        [FromQuery] string q, 
+        [FromQuery] int page = 1, 
+        [FromQuery] int perPage = 10,
+        CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(q))
+        {
+            return BadRequest(new { error = "Search query cannot be empty." });
+        }
+
+        var request = new SearchRequest
+        {
+            Query = q,
+            Page = Math.Max(1, page),
+            PerPage = Math.Min(Math.Max(1, perPage), 50)
+        };
+
+        return await Search(request, cancellationToken);
+    }
+
+    /// <summary>
+    /// Get search health status
+    /// </summary>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Health status</returns>
+    [HttpGet("health")]
+    public async Task<IActionResult> Health(CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var isHealthy = await _searchService.IsHealthyAsync(cancellationToken);
+
+            if (isHealthy)
+            {
+                return Ok(new { status = "healthy", timestamp = DateTime.UtcNow });
+            }
+            else
+            {
+                return StatusCode(503, new { status = "unhealthy", timestamp = DateTime.UtcNow });
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Health check failed");
+            return StatusCode(503, new { status = "error", timestamp = DateTime.UtcNow, error = ex.Message });
+        }
+    }
+}

EssentialCSharp.Web/Models/SearchModels.cs

@@ -0,0 +1,48 @@
+using System.Text.Json.Serialization;
+
+namespace EssentialCSharp.Web.Models;
+
+public class SearchDocument
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = string.Empty;
+
+    [JsonPropertyName("title")]
+    public string Title { get; set; } = string.Empty;
+
+    [JsonPropertyName("content")]
+    public string Content { get; set; } = string.Empty;
+
+    [JsonPropertyName("url")]
+    public string Url { get; set; } = string.Empty;
+
+    [JsonPropertyName("chapter")]
+    public string Chapter { get; set; } = string.Empty;
+
+    [JsonPropertyName("section")]
+    public string Section { get; set; } = string.Empty;
+
+    [JsonPropertyName("tags")]
+    public List<string> Tags { get; set; } = [];
+
+    [JsonPropertyName("created_at")]
+    public long CreatedAt { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+}
+
+public class SearchResult
+{
+    public List<SearchDocument> Results { get; set; } = [];
+    public int TotalCount { get; set; }
+    public int Page { get; set; }
+    public int PerPage { get; set; }
+    public double SearchTimeMs { get; set; }
+    public string Query { get; set; } = string.Empty;
+}
+
+public class SearchRequest
+{
+    public string Query { get; set; } = string.Empty;
+    public int Page { get; set; } = 1;
+    public int PerPage { get; set; } = 10;
+    public List<string> Filters { get; set; } = [];
+}

EssentialCSharp.Web/Program.cs

@@ -153,8 +153,15 @@ private static void Main(string[] args)
         builder.Services.AddSingleton<ISiteMappingService, SiteMappingService>();
         builder.Services.AddSingleton<IRouteConfigurationService, RouteConfigurationService>();
         builder.Services.AddHostedService<DatabaseMigrationService>();
+        builder.Services.AddHostedService<SearchIndexingHostedService>();
         builder.Services.AddScoped<IReferralService, ReferralService>();
 
+        // Add Typesense search services
+        builder.Services.Configure<TypesenseOptions>(
+            builder.Configuration.GetSection(TypesenseOptions.SectionName));
+        builder.Services.AddHttpClient<ITypesenseSearchService, TypesenseSearchService>();
+        builder.Services.AddScoped<IContentIndexingService, ContentIndexingService>();
+
         // Add AI Chat services
         if (!builder.Environment.IsDevelopment())
         {
@@ -198,6 +205,14 @@ private static void Main(string[] args)
                 rateLimiterOptions.QueueLimit = 0; // No queuing for anonymous users
             });
 
+            options.AddFixedWindowLimiter("SearchEndpoint", rateLimiterOptions =>
+            {
+                rateLimiterOptions.PermitLimit = 50; // search requests per window (higher limit for search)
+                rateLimiterOptions.Window = TimeSpan.FromMinutes(1); // minute window
+                rateLimiterOptions.QueueProcessingOrder = QueueProcessingOrder.OldestFirst;
+                rateLimiterOptions.QueueLimit = 0; // No queuing for immediate response
+            });
+
             // Custom response when rate limit is exceeded
             options.OnRejected = async (context, cancellationToken) =>
             {

EssentialCSharp.Web/Services/ContentIndexingService.cs

@@ -0,0 +1,186 @@
+using EssentialCSharp.Web.Models;
+using HtmlAgilityPack;
+using System.Text.RegularExpressions;
+
+namespace EssentialCSharp.Web.Services;
+
+public interface IContentIndexingService
+{
+    Task<bool> IndexAllContentAsync(CancellationToken cancellationToken = default);
+    Task<bool> IndexSiteMappingAsync(SiteMapping siteMapping, CancellationToken cancellationToken = default);
+}
+
+public class ContentIndexingService : IContentIndexingService
+{
+    private readonly ITypesenseSearchService _searchService;
+    private readonly ISiteMappingService _siteMappingService;
+    private readonly IWebHostEnvironment _environment;
+    private readonly ILogger<ContentIndexingService> _logger;
+
+    public ContentIndexingService(
+        ITypesenseSearchService searchService,
+        ISiteMappingService siteMappingService,
+        IWebHostEnvironment environment,
+        ILogger<ContentIndexingService> logger)
+    {
+        _searchService = searchService;
+        _siteMappingService = siteMappingService;
+        _environment = environment;
+        _logger = logger;
+    }
+
+    public async Task<bool> IndexAllContentAsync(CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            _logger.LogInformation("Starting to index all content");
+
+            // Initialize the collection if it doesn't exist
+            if (!await _searchService.InitializeCollectionAsync(cancellationToken))
+            {
+                _logger.LogError("Failed to initialize Typesense collection");
+                return false;
+            }
+
+            var documents = new List<SearchDocument>();
+
+            foreach (var siteMapping in _siteMappingService.SiteMappings)
+            {
+                var document = await CreateSearchDocumentAsync(siteMapping);
+                if (document != null)
+                {
+                    documents.Add(document);
+                }
+            }
+
+            if (documents.Count > 0)
+            {
+                var success = await _searchService.IndexDocumentsAsync(documents, cancellationToken);
+                _logger.LogInformation("Indexed {Count} documents, success: {Success}", documents.Count, success);
+                return success;
+            }
+
+            _logger.LogWarning("No documents to index");
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to index all content");
+            return false;
+        }
+    }
+
+    public async Task<bool> IndexSiteMappingAsync(SiteMapping siteMapping, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var document = await CreateSearchDocumentAsync(siteMapping);
+            if (document == null)
+            {
+                return false;
+            }
+
+            return await _searchService.IndexDocumentAsync(document, cancellationToken);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to index site mapping {Key}", siteMapping.PrimaryKey);
+            return false;
+        }
+    }
+
+    private async Task<SearchDocument?> CreateSearchDocumentAsync(SiteMapping siteMapping)
+    {
+        try
+        {
+            var filePath = Path.Combine(_environment.ContentRootPath, Path.Combine(siteMapping.PagePath));
+            if (!File.Exists(filePath))
+            {
+                _logger.LogWarning("File not found: {FilePath}", filePath);
+                return null;
+            }
+
+            var htmlContent = await File.ReadAllTextAsync(filePath);
+            var doc = new HtmlDocument();
+            doc.LoadHtml(htmlContent);
+
+            // Extract content from body
+            var bodyNode = doc.DocumentNode.SelectSingleNode("//body");
+            if (bodyNode == null)
+            {
+                _logger.LogWarning("No body content found in {FilePath}", filePath);
+                return null;
+            }
+
+            // Remove script and style elements
+            var scriptsAndStyles = bodyNode.SelectNodes("//script | //style");
+            if (scriptsAndStyles != null)
+            {
+                foreach (var node in scriptsAndStyles)
+                {
+                    node.Remove();
+                }
+            }
+
+            // Extract plain text content
+            var textContent = bodyNode.InnerText;
+            var cleanContent = CleanTextContent(textContent);
+
+            // Create tags based on the content
+            var tags = new List<string>();
+            if (!string.IsNullOrEmpty(siteMapping.ChapterTitle))
+            {
+                tags.Add($"chapter-{siteMapping.ChapterNumber}");
+            }
+
+            // Extract URL from the first key
+            var url = $"/{siteMapping.Keys.First()}";
+            if (!string.IsNullOrEmpty(siteMapping.AnchorId))
+            {
+                url += $"#{siteMapping.AnchorId}";
+            }
+
+            return new SearchDocument
+            {
+                Id = siteMapping.PrimaryKey,
+                Title = siteMapping.RawHeading ?? siteMapping.ChapterTitle ?? "Unknown",
+                Content = cleanContent,
+                Url = url,
+                Chapter = $"Chapter {siteMapping.ChapterNumber}: {siteMapping.ChapterTitle}",
+                Section = siteMapping.RawHeading ?? string.Empty,
+                Tags = tags,
+                CreatedAt = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
+            };
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to create search document for {Key}", siteMapping.PrimaryKey);
+            return null;
+        }
+    }
+
+    private static string CleanTextContent(string htmlText)
+    {
+        if (string.IsNullOrEmpty(htmlText))
+        {
+            return string.Empty;
+        }
+
+        // Decode HTML entities
+        var decodedText = HtmlEntity.DeEntitize(htmlText);
+
+        // Remove extra whitespace and normalize line breaks
+        var cleanText = Regex.Replace(decodedText, @"\s+", " ");
+
+        // Remove leading/trailing whitespace
+        cleanText = cleanText.Trim();
+
+        // Limit content length for search indexing (Typesense has limits)
+        if (cleanText.Length > 10000)
+        {
+            cleanText = cleanText[..10000] + "...";
+        }
+
+        return cleanText;
+    }
+}