Merge pull request #660 from iceljc/master

iceljc · web-flow · commit a0e7117e2959 · 2024-09-30T18:27:23.000-05:00
refine sql
diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs
@@ -24,7 +24,7 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         state.SetState("max_tokens", "4096");
         var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
-        var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var collectionName = knowledgeSettings.Default.CollectionName;
 
         // Get knowledge from vectordb
         var hooks = _services.GetServices<IKnowledgeHook>();
diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs
@@ -17,16 +17,14 @@ public SecondaryStagePlanFn(IServiceProvider services, ILogger<SecondaryStagePla
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
-        var fn = _services.GetRequiredService<IRoutingService>();
         var agentService = _services.GetRequiredService<IAgentService>();
         var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
         var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
         var states = _services.GetRequiredService<IConversationStateService>();
 
         var msgSecondary = RoleDialogModel.From(message);
-        var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var collectionName = knowledgeSettings.Default.CollectionName;
         var planPrimary = states.GetState("planning_result");
-        var taskPrimary = states.GetState("requirement_detail");
 
         var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
  
@@ -35,8 +33,8 @@ public async Task<bool> Execute(RoleDialogModel message)
         {
             Confidence = 0.6f
         });
-        var knowledgeResults = "";
-        knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
+
+        var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
 
         // Get second stage planning prompt
         var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
@@ -45,7 +43,7 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         var plannerAgent = new Agent
         {
-            Id = string.Empty,
+            Id = BuiltInAgentId.Planner,
             Name = "planning_2nd",
             Instruction = secondPlanningPrompt,
             TemplateDict = new Dictionary<string, object>(),
diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs
@@ -33,15 +33,21 @@ public async Task<bool> Execute(RoleDialogModel message)
         var states = _services.GetRequiredService<IConversationStateService>();
         var steps = states.GetState("planning_result").JsonArrayContent<SecondStagePlan>();
         var allTables = new List<string>();
-        var ddlStatements = "";
+        var ddlStatements = string.Empty;
         var relevantKnowledge = states.GetState("planning_result");
         var dictionaryItems = states.GetState("dictionary_items");
+        var items = new List<string>();
+        if (!string.IsNullOrWhiteSpace(dictionaryItems))
+        {
+            items = JsonSerializer.Deserialize<List<string>>(dictionaryItems);
+        }
 
         foreach (var step in steps)
         {
             allTables.AddRange(step.Tables);
         }
         var distinctTables = allTables.Distinct().ToList();
+
         foreach (var table in distinctTables)
         {
             var msgCopy = RoleDialogModel.From(message);
@@ -54,7 +60,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         }
 
         // Summarize and generate query
-        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements);
+        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, items, ddlStatements);
         _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}");
 
         var plannerAgent = new Agent
@@ -74,7 +80,7 @@ await HookEmitter.Emit<IPlanningHook>(_services, x =>
         return true;
     }
 
-    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement)
+    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, IEnumerable<string> dictionaryItems, string ddlStatement)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
@@ -92,9 +98,9 @@ await HookEmitter.Emit<IPlanningHook>(_services, async x =>
         return render.Render(template, new Dictionary<string, object>
         {
             { "task_description", taskDescription },
-            { "summary_requirements", string.Join("\r\n",additionalRequirements) },
+            { "summary_requirements", string.Join("\r\n", additionalRequirements) },
             { "relevant_knowledges", relevantKnowledge },
-            { "dictionary_items", dictionaryItems },
+            { "dictionary_items", string.Join("\r\n\r\n", dictionaryItems) },
             { "table_structure", ddlStatement },
         });
     }
diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json
@@ -11,7 +11,7 @@
   "profiles": [ "planning" ],
   "utilities": [ "two-stage-planner" ],
   "llmConfig": {
-    "provider": "azure-openai",
+    "provider": "openai",
     "model": "gpt-4o",
     "max_recursion_depth": 10
   }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
@@ -11,8 +11,11 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\**" />
     <Compile Remove="packages\**" />
+    <EmbeddedResource Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\**" />
     <EmbeddedResource Remove="packages\**" />
+    <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\**" />
     <None Remove="packages\**" />
   </ItemGroup>
 
@@ -80,8 +83,4 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\" />
-  </ItemGroup>
-
 </Project>
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs
@@ -19,6 +19,6 @@ public SqlDriverController(IServiceProvider services)
     public async Task<bool> ImportDbKnowledge(ImportDbKnowledgeRequest request)
     {
         var dbKnowledge = _services.GetRequiredService<DbKnowledgeService>();
-        return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema);
+        return await dbKnowledge.Import(request);
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
@@ -44,7 +44,6 @@ private IEnumerable<dynamic> RunQueryInSqlServer(string[] sqlTexts)
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
-        var dictionary = new Dictionary<string, object>();
         return connection.Query(string.Join("\r\n", sqlTexts));
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs
@@ -1,5 +1,4 @@
 using BotSharp.Plugin.SqlDriver.Models;
-using Fluid.Ast.BinaryExpressions;
 using Microsoft.Data.SqlClient;
 using Microsoft.Extensions.Logging;
 using MySqlConnector;
@@ -37,18 +36,14 @@ public async Task<bool> Execute(RoleDialogModel message)
         };
     
         message.Content = string.Join("\r\n\r\n", tableDdls);
-
-        //var states = _services.GetRequiredService<IConversationStateService>();
-        //states.SetState($"table_definition_{args.Table}", message.Content);
-
         return true;
     }
 
     private List<string> GetDdlFromMySql(string[] tables)
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         var tableDdls = new List<string>();
-        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
+        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString);
         connection.Open();
 
         foreach (var table in tables)
@@ -76,7 +71,6 @@ private List<string> GetDdlFromMySql(string[] tables)
         }
 
         connection.Close();
-
         return tableDdls;
     }
 
@@ -92,27 +86,27 @@ private List<string> GetDdlFromSqlServer(string[] tables)
             try
             {
                 var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}';
-DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' (';
-
-SELECT @SQL = @SQL + '
-    ' + COLUMN_NAME + ' ' + 
-    DATA_TYPE + 
-    CASE 
-        WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' 
-            THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'
-        WHEN DATA_TYPE IN ('decimal', 'numeric')
-            THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'
-        ELSE ''
-    END + ' ' + 
-    CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' 
-FROM INFORMATION_SCHEMA.COLUMNS 
-WHERE TABLE_NAME = @TableName
-ORDER BY ORDINAL_POSITION;
-
--- Remove the last comma and add closing parenthesis
-SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');';
-
-SELECT @SQL;";
+                            DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' (';
+
+                            SELECT @SQL = @SQL + '
+                                ' + COLUMN_NAME + ' ' + 
+                                DATA_TYPE + 
+                                CASE 
+                                    WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' 
+                                        THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'
+                                    WHEN DATA_TYPE IN ('decimal', 'numeric')
+                                        THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'
+                                    ELSE ''
+                                END + ' ' + 
+                                CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' 
+                            FROM INFORMATION_SCHEMA.COLUMNS 
+                            WHERE TABLE_NAME = @TableName
+                            ORDER BY ORDINAL_POSITION;
+
+                            -- Remove the last comma and add closing parenthesis
+                            SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');';
+
+                            SELECT @SQL;";
 
                 using var command = new SqlCommand(sql, connection);
                 using var reader = command.ExecuteReader();
@@ -129,7 +123,6 @@ FROM INFORMATION_SCHEMA.COLUMNS
         }
 
         connection.Close();
-
         return tableDdls;
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs
@@ -1,14 +1,9 @@
-using Azure;
 using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Routing;
-using BotSharp.Core.Agents.Services;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.SqlDriver.Models;
 using MySqlConnector;
-using System.Text.RegularExpressions;
 using static Dapper.SqlMapper;
-using static System.Net.Mime.MediaTypeNames;
 
 namespace BotSharp.Plugin.SqlDriver.Functions;
 
@@ -35,15 +30,16 @@ public async Task<bool> Execute(RoleDialogModel message)
         var agentService = _services.GetRequiredService<IAgentService>();
         var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
         var dictionarySqlPrompt = await GetDictionarySQLPrompt(args.SqlStatement, msgCopy.Content);
-        var plannerAgent = new Agent
+        var agent = new Agent
         {
-            Id = string.Empty,
+            Id = message.CurrentAgentId ?? string.Empty,
             Name = "sqlDriver_DictionarySearch",
             Instruction = dictionarySqlPrompt,
             TemplateDict = new Dictionary<string, object>(),
             LlmConfig = currentAgent.LlmConfig
         };
-        var response = await GetAiResponse(plannerAgent);
+
+        var response = await GetAiResponse(agent);
         args = JsonSerializer.Deserialize<LookupDictionary>(response.Content);
 
         // check if need to instantely
@@ -59,13 +55,24 @@ public async Task<bool> Execute(RoleDialogModel message)
         {
             message.Content = JsonSerializer.Serialize(result);
         }
+
         var states = _services.GetRequiredService<IConversationStateService>();
-        var dictionaryItems = states.GetState("dictionary_items", "");
-        dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n";
-        states.SetState("dictionary_items", dictionaryItems);
+        var dictionaryItems = states.GetState("dictionary_items");
+        var newItem = BuildDictionaryItem(args.Table, args.Reason, message.Content);
+
+        var items = new List<string>();
+        if (!string.IsNullOrWhiteSpace(dictionaryItems))
+        {
+            items = JsonSerializer.Deserialize<List<string>>(dictionaryItems);
+        }
+
+        items.Add(newItem);
+        //dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n";
+        states.SetState("dictionary_items", JsonSerializer.Serialize(items));
 
         return true;
     }
+
     private async Task<string> GetDictionarySQLPrompt(string originalSql, string tableStructure)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -83,15 +90,39 @@ private async Task<string> GetDictionarySQLPrompt(string originalSql, string tab
             { "response_format", responseFormat }
         });
     }
-    private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
+
+    private async Task<RoleDialogModel> GetAiResponse(Agent agent)
     {
         var text = "Check and correct the SQL statement.";
         var message = new RoleDialogModel(AgentRole.User, text);
 
         var completion = CompletionProvider.GetChatCompletion(_services,
-            provider: plannerAgent.LlmConfig.Provider,
-            model: plannerAgent.LlmConfig.Model);
+            provider: agent.LlmConfig.Provider,
+            model: agent.LlmConfig.Model);
+
+        return await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
+    }
+
+    private string BuildDictionaryItem(string? table, string? reason, string? result)
+    {
+        var res = new List<string>();
+        if (!string.IsNullOrWhiteSpace(table))
+        {
+            res.Add($"Table: {table}");
+        }
+
+        if (!string.IsNullOrWhiteSpace(reason))
+        {
+            res.Add($"Reason: {reason}");
+        }
+
+        if (!string.IsNullOrWhiteSpace(result))
+        {
+            res.Add($"Result: {result}");
+        }
+
+        if (res.IsNullOrEmpty()) return string.Empty;
 
-        return await completion.GetChatCompletions(plannerAgent, new List<RoleDialogModel> { message });
+        return string.Join("\r\n", res);
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs
@@ -2,7 +2,6 @@
 using BotSharp.Abstraction.Agents.Settings;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Repositories;
-using System.Collections.Generic;
 
 namespace BotSharp.Plugin.SqlDriver.Hooks;
 
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
@@ -1,7 +1,6 @@
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.Planning;
 using BotSharp.Abstraction.Routing;
-using BotSharp.Core.Agents.Services;
 using BotSharp.Core.Infrastructures;
 
 namespace BotSharp.Plugin.SqlDriver.Hooks;
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs
@@ -5,11 +5,11 @@ namespace BotSharp.Plugin.SqlDriver.Models;
 public class LookupDictionary
 {
     [JsonPropertyName("sql_statement")]
-    public string SqlStatement { get; set; }
+    public string? SqlStatement { get; set; }
 
     [JsonPropertyName("reason")]
-    public string Reason { get; set; }
+    public string? Reason { get; set; }
 
     [JsonPropertyName("table")]
-    public string Table { get; set; }
+    public string? Table { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs
@@ -16,4 +16,7 @@ public class ImportDbKnowledgeRequest : RequestBase
 {
     [JsonPropertyName("schema")]
     public string Schema { get; set; }
+
+    [JsonPropertyName("knowledgebase_collection")]
+    public string KnowledgebaseCollection { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`"profiles": [ "planning" ],`
`12`	`12`	`"utilities": [ "two-stage-planner" ],`
`13`	`13`	`"llmConfig": {`
`14`		`- "provider": "azure-openai",`
	`14`	`+ "provider": "openai",`
`15`	`15`	`"model": "gpt-4o",`
`16`	`16`	`"max_recursion_depth": 10`
`17`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,6 @@ public SqlDriverController(IServiceProvider services)`
`19`	`19`	`public async Task<bool> ImportDbKnowledge(ImportDbKnowledgeRequest request)`
`20`	`20`	`{`
`21`	`21`	`var dbKnowledge = _services.GetRequiredService<DbKnowledgeService>();`
`22`		`- return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema);`
	`22`	`+ return await dbKnowledge.Import(request);`
`23`	`23`	`}`
`24`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,6 @@ private IEnumerable<dynamic> RunQueryInSqlServer(string[] sqlTexts)`
`44`	`44`	`{`
`45`	`45`	`var settings = _services.GetRequiredService<SqlDriverSetting>();`
`46`	`46`	`using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);`
`47`		`- var dictionary = new Dictionary<string, object>();`
`48`	`47`	`return connection.Query(string.Join("\r\n", sqlTexts));`
`49`	`48`	`}`
`50`	`49`	`}`