Merge pull request #670 from Joannall/master

improve data dictionary lookup and plan summary

Author: Oceania2018

src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs

@@ -51,6 +51,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
         var conv = _serviceProvider.GetRequiredService<IConversationService>();
 
+
         if (_excelMimeTypes.IsNullOrEmpty())
         {
             _excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
@@ -70,7 +71,10 @@ public async Task<bool> Execute(RoleDialogModel message)
         else
         {
             var resultList = GetResponeFromDialogs(dialogs);
+            var states = _serviceProvider.GetRequiredService<IConversationStateService>();
+
             message.Content = GenerateSqlExecutionSummary(resultList);
+            states.SetState("excel_import_result",message.Content);
         }
         return true;
     }

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs

@@ -1,4 +1,5 @@
 using System.Data;
+using BotSharp.Abstraction.Routing;
 using BotSharp.Plugin.ExcelHandler.Helpers.MySql;
 using BotSharp.Plugin.ExcelHandler.Models;
 using MySql.Data.MySqlClient;
@@ -9,17 +10,19 @@ namespace BotSharp.Plugin.ExcelHandler.Services
     public class MySqlService : IMySqlService
     {
         private readonly IMySqlDbHelper _mySqlDbHelpers;
-
+        private readonly IServiceProvider _services;
         private double _excelRowSize = 0;
         private double _excelColumnSize = 0;
         private string _tableName = "tempTable";
+        private string _database = "";
         private string _currentFileName = string.Empty;
         private List<string> _headerColumns = new List<string>();
         private List<string> _columnTypes = new List<string>();
 
-        public MySqlService(IMySqlDbHelper mySqlDbHelpers)
+        public MySqlService(IMySqlDbHelper mySqlDbHelpers, IServiceProvider services)
         {
             _mySqlDbHelpers = mySqlDbHelpers;
+            _services = services;
         }
 
         public bool DeleteTableSqlQuery()
@@ -185,10 +188,12 @@ private string ParseSheetData(ISheet singleSheet)
         {
             try
             {
-                _tableName = $"excel_{sheet.SheetName}";
+                var routing = _services.GetRequiredService<IRoutingContext>();
+                _tableName = $"excel_{routing.ConversationId.Split('-').Last()}_{sheet.SheetName}";
                 _headerColumns = ParseSheetColumn(sheet);
                 string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null ,true);
                 ExecuteSqlQueryForInsertion(createTableSql);
+                createTableSql = createTableSql.Replace(_tableName, $"{_database}.{_tableName}");
                 return (true, createTableSql);
             }
             catch (Exception ex)
@@ -210,21 +215,26 @@ private List<string> ParseSheetColumn(ISheet sheet)
         }
         private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
         {
-            var createTableSql = $"CREATE TABLE if not exists {tableName} ( ";
-
             _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes;
 
-            headerColumns = headerColumns.Select((x, i) => $"`{x}`" + $" {_columnTypes[i]}").ToList();
-            createTableSql += string.Join(", ", headerColumns);
+            /*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
+            {
+                headerColumns.Insert(0, "Id");
+                _columnTypes?.Insert(0, "INT UNSIGNED AUTO_INCREMENT");
+            }*/
+
+            var createTableSql = $"CREATE TABLE if not exists {tableName} ( \n";
+            createTableSql += string.Join(", \n", headerColumns.Select((x, i) => $"`{x}` {_columnTypes[i]}"));
+            var indexSql = string.Join(", \n", headerColumns.Select(x => $"KEY `idx_{tableName}_{x}` (`{x}`)"));
+            createTableSql += $", \n{indexSql}\n);";
 
-            string engine = isMemory ? "ENGINE=MEMORY" : "";
-            createTableSql += $") {engine};";
             return createTableSql;
         }
 
         public void ExecuteSqlQueryForInsertion(string sqlQuery)
         {
             using var connection = _mySqlDbHelpers.GetDbConnection();
+            _database = connection.Database;
             using (MySqlCommand cmd = new MySqlCommand(sqlQuery, connection))
             {
                 cmd.ExecuteNonQuery();

src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Planning;
 using BotSharp.Plugin.Planner.TwoStaging;
 using BotSharp.Plugin.Planner.TwoStaging.Models;
+using Microsoft.EntityFrameworkCore.Metadata.Internal;
 
 namespace BotSharp.Plugin.Planner.Functions;
 
@@ -36,26 +37,24 @@ public async Task<bool> Execute(RoleDialogModel message)
         var ddlStatements = string.Empty;
         var relevantKnowledge = states.GetState("planning_result");
         var dictionaryItems = states.GetState("dictionary_items");
+        var excelImportResult = states.GetState("excel_import_result");
 
         foreach (var step in steps)
         {
             allTables.AddRange(step.Tables);
         }
         var distinctTables = allTables.Distinct().ToList();
 
-        foreach (var table in distinctTables)
+        var msgCopy = RoleDialogModel.From(message);
+        msgCopy.FunctionArgs = JsonSerializer.Serialize(new
         {
-            var msgCopy = RoleDialogModel.From(message);
-            msgCopy.FunctionArgs = JsonSerializer.Serialize(new  
-            { 
-                table = table,
-            });
-            await fn.InvokeFunction("sql_table_definition", msgCopy);
-            ddlStatements += "\r\n" + msgCopy.Content;
-        }
+            tables = distinctTables,
+        });
+        await fn.InvokeFunction("sql_table_definition", msgCopy);
+        ddlStatements += "\r\n" + msgCopy.Content;
 
         // Summarize and generate query
-        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements);
+        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult);
         _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}");
 
         var plannerAgent = new Agent
@@ -75,7 +74,7 @@ await HookEmitter.Emit<IPlanningHook>(_services, x =>
         return true;
     }
 
-    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement)
+    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
@@ -97,6 +96,7 @@ await HookEmitter.Emit<IPlanningHook>(_services, async x =>
             { "relevant_knowledges", relevantKnowledge },
             { "dictionary_items", dictionaryItems },
             { "table_structure", ddlStatement },
+            { "excel_import_result",excelImportResult }
         });
     }
     private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json

@@ -10,10 +10,10 @@
       },
       "questions": {
         "type": "array",
-        "description": "User requirements in detail, don't miss any information especially for those line items, values and numbers.",
+        "description": "Rephrase user requirements in details and in multiple ways, don't miss any information especially for those line items, values and numbers.",
         "items": {
           "type": "string",
-          "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge."
+          "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge.One question should contain only one main topic."
         }
       }
     },

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid

@@ -18,3 +18,7 @@ Dictionary Items:
 =====
 Table Structure:
 {{ table_structure }}
+
+=====
+Attached Excel Information: 
+{{ excel_import_result }}

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs

@@ -23,7 +23,7 @@ public GetTableDefinitionFn(
     public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
-        var tables = new string[] { args.Table };
+        var tables = args.Tables;
         var agentService = _services.GetRequiredService<IAgentService>();
         var settings = _services.GetRequiredService<SqlDriverSetting>();
 

src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs

@@ -10,8 +10,8 @@ public class SqlStatement
     [JsonPropertyName("reason")]
     public string Reason { get; set; } = null!;
 
-    [JsonPropertyName("table")]
-    public string Table { get; set; } = null!;
+    [JsonPropertyName("tables")]
+    public string[] Tables { get; set; } = null!;
 
     [JsonPropertyName("parameters")]
     public SqlParameter[] Parameters { get; set; } = [];

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json

@@ -12,11 +12,14 @@
         "type": "string",
         "description": "the reason why you need to call sql_dictionary_lookup"
       },
-      "table": {
-        "type": "string",
-        "description": "table name"
-      }
-    },
-    "required": [ "sql_statement", "reason", "table" ]
+      "tables": {
+        "type": "array",
+        "description": "all related tables",
+        "items": {
+          "type": "string",
+          "description": "table name"
+        }
+      },
+    "required": [ "sql_statement", "reason", "tables" ]
   }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid

@@ -1 +1 @@
-Output in human readable format.
+Output in human readable format. If there is large amount of information, shape it in tabular.