From 3e718f25f237aad00e6721a62621867f054a953a Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Fri, 17 Oct 2025 19:14:31 +0530 Subject: [PATCH 01/26] boilerplate api --- src/intugle/conceptual_search/__init__.py | 0 src/intugle/conceptual_search/engine.py | 52 +++++++++++++++++++++++ src/intugle/conceptual_search/models.py | 25 +++++++++++ src/intugle/data_product.py | 20 +++++++++ 4 files changed, 97 insertions(+) create mode 100644 src/intugle/conceptual_search/__init__.py create mode 100644 src/intugle/conceptual_search/engine.py create mode 100644 src/intugle/conceptual_search/models.py diff --git a/src/intugle/conceptual_search/__init__.py b/src/intugle/conceptual_search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/intugle/conceptual_search/engine.py b/src/intugle/conceptual_search/engine.py new file mode 100644 index 0000000..c1d099d --- /dev/null +++ b/src/intugle/conceptual_search/engine.py @@ -0,0 +1,52 @@ +# src/intugle/conceptual_search/engine.py +from intugle.core import settings +from intugle.parser.manifest import ManifestLoader + +from .models import ConceptualAttribute, ConceptualPlan, MappedAttribute, MappedPlan + + +class ConceptualSearch: + def __init__(self, project_base: str = settings.PROJECT_BASE): + """Initializes by loading the manifest to provide context for the agents.""" + self.manifest_loader = ManifestLoader(project_base) + self.manifest_loader.load() + self.manifest = self.manifest_loader.manifest + # Initialize your LLM clients for the agents here + + def create_plan(self, concept: str) -> ConceptualPlan: + """ + Agent 1: Takes a high-level concept and generates a conceptual plan. + """ + print(f"Agent 1: Generating conceptual plan for '{concept}'...") + # --- YOUR AGENT 1 LOGIC GOES HERE --- + # This logic will use an LLM to transform the 'concept' string + # into a list of ConceptualAttribute objects. + # For this example, we'll use mock data. + + mock_attributes = [ + ConceptualAttribute(name="Patient Name", description="The full name of the patient", logic="Concatenate first and last names"), + ConceptualAttribute(name="Total Claims", description="The total number of claims filed by the patient", logic="Count all claims associated with the patient ID") + ] + + plan = ConceptualPlan(attributes=mock_attributes) + print("Agent 1: Plan created successfully.") + return plan + + def map_plan_to_columns(self, plan: ConceptualPlan) -> MappedPlan: + """ + Agent 2: Takes a conceptual plan and maps it to physical columns. + """ + print("Agent 2: Mapping conceptual plan to semantic layer columns...") + # --- YOUR AGENT 2 LOGIC GOES HERE --- + # This logic uses an LLM, providing the conceptual plan and the + # manifest (self.manifest) as context to find the real column_ids. + # For this example, we'll use mock data. + + mapped_attributes = [ + MappedAttribute(name="Patient Name", description="The full name of the patient", logic="Concatenate first and last names", column_id="patients.first"), + MappedAttribute(name="Total Claims", description="The total number of claims filed by the patient", logic="Count all claims associated with the patient ID", column_id="claims.id", measure_func="COUNT") + ] + + mapped_plan = MappedPlan(attributes=mapped_attributes) + print("Agent 2: Mapping complete.") + return mapped_plan diff --git a/src/intugle/conceptual_search/models.py b/src/intugle/conceptual_search/models.py new file mode 100644 index 0000000..d842581 --- /dev/null +++ b/src/intugle/conceptual_search/models.py @@ -0,0 +1,25 @@ +# src/intugle/conceptual_search/models.py +from typing import Optional, List +from pydantic import BaseModel, Field + +class ConceptualAttribute(BaseModel): + """An abstract attribute for the desired data product.""" + name: str = Field(description="A business-friendly name for the column, e.g., 'Patient Age'") + description: str = Field(description="A detailed explanation of what this attribute represents.") + logic: str = Field(description="A natural language description of the calculation, e.g., 'Calculate the age based on the date of birth'") + +class ConceptualPlan(BaseModel): + """The conceptual plan generated by Agent 1, which can be reviewed by the user.""" + attributes: List[ConceptualAttribute] + + def to_dict(self): + return self.model_dump() + +class MappedAttribute(ConceptualAttribute): + """A conceptual attribute that has been mapped to a physical column.""" + column_id: str = Field(description="The fully qualified column ID from the semantic layer, e.g., 'patients.birth_date'") + measure_func: Optional[str] = Field(default=None, description="The aggregation function if any, e.g., 'COUNT', 'SUM'") + +class MappedPlan(BaseModel): + """The final plan after Agent 2 has mapped attributes to columns.""" + attributes: List[MappedAttribute] diff --git a/src/intugle/data_product.py b/src/intugle/data_product.py index a101b2d..e2376cf 100644 --- a/src/intugle/data_product.py +++ b/src/intugle/data_product.py @@ -2,6 +2,7 @@ from intugle.adapters.factory import AdapterFactory from intugle.analysis.models import DataSet +from intugle.conceptual_search.models import MappedPlan from intugle.core import settings from intugle.libs.smart_query_generator import SmartQueryGenerator from intugle.libs.smart_query_generator.models.models import ETLModel, FieldDetailsModel, LinkModel @@ -32,6 +33,25 @@ def __init__(self, project_base: str = settings.PROJECT_BASE): self.load_all() + @staticmethod + def etl_from_mapped_plan(plan: MappedPlan, product_name: str) -> dict: + """Creates an ETLModel dictionary from a mapped conceptual plan.""" + fields = [] + for attr in plan.attributes: + field = { + "id": attr.column_id, + "name": attr.name, + } + if attr.measure_func: + field["category"] = "measure" + field["measure_func"] = attr.measure_func.lower() + fields.append(field) + + return { + "name": product_name, + "fields": fields + } + def load_all(self): sources = self.manifest.sources for source in sources.values(): From 09e8367201180bc63103c30d7182f9d6af3ba800 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Fri, 17 Oct 2025 20:11:18 +0530 Subject: [PATCH 02/26] refactored project base --- src/intugle/analysis/models.py | 9 +++-- src/intugle/core/settings.py | 5 ++- src/intugle/data_product.py | 6 ++-- src/intugle/link_predictor/predictor.py | 10 +++--- src/intugle/mcp/manifest.py | 2 +- src/intugle/parser/manifest.py | 36 +++++++++---------- src/intugle/semantic_model.py | 5 +-- src/intugle/semantic_search.py | 6 ++-- src/intugle/utils/files.py | 2 +- tests/query_generator/test_query_generator.py | 2 +- tests/semantic_search/test_semantic_search.py | 2 +- 11 files changed, 47 insertions(+), 38 deletions(-) diff --git a/src/intugle/analysis/models.py b/src/intugle/analysis/models.py index c83b834..b3852e9 100644 --- a/src/intugle/analysis/models.py +++ b/src/intugle/analysis/models.py @@ -50,7 +50,7 @@ def __init__(self, data: DataSetData, name: str): self.columns: Dict[str, Column] = {} # A convenience map for quick column lookup # Check if a YAML file exists and load it - file_path = os.path.join(settings.PROJECT_BASE, f"{self.name}.yml") + file_path = os.path.join(settings.MODELS_DIR, f"{self.name}.yml") if os.path.exists(file_path): print(f"Found existing YAML for '{self.name}'. Checking for staleness.") self.load_from_yaml(file_path) @@ -315,7 +315,10 @@ def run(self, domain: str, save: bool = True) -> 'DataSet': def save_yaml(self, file_path: Optional[str] = None) -> None: if file_path is None: file_path = f"{self.name}.yml" - file_path = os.path.join(settings.PROJECT_BASE, file_path) + + # Ensure the models directory exists + os.makedirs(settings.MODELS_DIR, exist_ok=True) + file_path = os.path.join(settings.MODELS_DIR, file_path) details = self.adapter.get_details(self.data) self.source_table_model.details = details @@ -352,7 +355,7 @@ def reload_from_yaml(self, file_path: Optional[str] = None) -> None: """Forces a reload from a YAML file, bypassing staleness checks.""" if file_path is None: file_path = f"{self.name}.yml" - file_path = os.path.join(settings.PROJECT_BASE, file_path) + file_path = os.path.join(settings.MODELS_DIR, file_path) with open(file_path, "r") as f: yaml_data = yaml.safe_load(f) diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index d4aef44..4b2f789 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -19,7 +19,7 @@ def create_project_base_if_not_exists(): """Create the base path directory if it does not exist.""" - project_base = Path(os.getenv("VSCODE_WORKSPACE", os.getcwd()), "models") + project_base = Path(os.getenv("VSCODE_WORKSPACE", os.getcwd()), "intugle") if not project_base.exists(): project_base.mkdir(parents=True, exist_ok=True) return str(project_base) @@ -40,6 +40,9 @@ class Settings(BaseSettings): DI_MODEL_VERSION: str = "13052023" PROJECT_BASE: str = create_project_base_if_not_exists() + MODELS_DIR_NAME: str = "models" + MODELS_DIR: str = os.path.join(PROJECT_BASE, MODELS_DIR_NAME) + PROFILES_PATH: str = os.path.join(os.getcwd(), "profiles.yml") PROFILES: dict = load_profiles_configuration(PROFILES_PATH) diff --git a/src/intugle/data_product.py b/src/intugle/data_product.py index e2376cf..70cf928 100644 --- a/src/intugle/data_product.py +++ b/src/intugle/data_product.py @@ -16,12 +16,12 @@ class DataProduct: """Generates data products based on the manifest and ETL configurations.""" - def __init__(self, project_base: str = settings.PROJECT_BASE): - self.manifest_loader = ManifestLoader(project_base) + def __init__(self, models_dir_path: str = settings.MODELS_DIR): + self.manifest_loader = ManifestLoader(models_dir_path) self.manifest_loader.load() self.manifest = self.manifest_loader.manifest - self.project_base = project_base + self.models_dir_path = models_dir_path self.field_details = self.get_all_field_details() diff --git a/src/intugle/link_predictor/predictor.py b/src/intugle/link_predictor/predictor.py index 33ea723..2580a5d 100644 --- a/src/intugle/link_predictor/predictor.py +++ b/src/intugle/link_predictor/predictor.py @@ -149,13 +149,13 @@ def predict(self, filename: str = None, save: bool = False, force_recreate: bool if filename is None: filename = settings.RELATIONSHIPS_FILE - relationships_file = os.path.join(settings.PROJECT_BASE, filename) + relationships_file = os.path.join(settings.MODELS_DIR, filename) if not force_recreate and os.path.exists(relationships_file): is_stale = False relationships_mtime = os.path.getmtime(relationships_file) for dataset in self.datasets.values(): - dataset_yml = os.path.join(settings.PROJECT_BASE, f"{dataset.name}.yml") + dataset_yml = os.path.join(settings.MODELS_DIR, f"{dataset.name}.yml") if os.path.exists(dataset_yml) and os.path.getmtime(dataset_yml) > relationships_mtime: is_stale = True break @@ -208,7 +208,8 @@ def show_graph(self): join.plot_graph(graph) def save_yaml(self, file_path: str) -> None: - file_path = os.path.join(settings.PROJECT_BASE, file_path) + os.makedirs(settings.MODELS_DIR, exist_ok=True) + file_path = os.path.join(settings.MODELS_DIR, file_path) if len(self.links) == 0: raise NoLinksFoundError("No links found to save.") @@ -245,7 +246,8 @@ def load_from_yaml(self, file_path: str) -> None: class LinkPredictionSaver: @classmethod def save_yaml(cls, result: LinkPredictionResult, file_path: str) -> None: - file_path = os.path.join(settings.PROJECT_BASE, file_path) + os.makedirs(settings.MODELS_DIR, exist_ok=True) + file_path = os.path.join(settings.MODELS_DIR, file_path) links = result.links diff --git a/src/intugle/mcp/manifest.py b/src/intugle/mcp/manifest.py index bc96047..7c94efb 100644 --- a/src/intugle/mcp/manifest.py +++ b/src/intugle/mcp/manifest.py @@ -2,6 +2,6 @@ from intugle.core import settings from intugle.parser.manifest import ManifestLoader -manifest_loader = ManifestLoader(settings.PROJECT_BASE) +manifest_loader = ManifestLoader(settings.MODELS_DIR) manifest_loader.load() diff --git a/src/intugle/parser/manifest.py b/src/intugle/parser/manifest.py index c4b6a52..14211bc 100644 --- a/src/intugle/parser/manifest.py +++ b/src/intugle/parser/manifest.py @@ -16,29 +16,29 @@ class FileReaderFromFileSystem: """Reads files from the file system and provides methods to filter and read YAML files.""" - def __init__(self, project_base_path: str): - """Initializes the file reader with the project base path. + def __init__(self, models_dir_path: str): + """Initializes the file reader with the models directory path. Args: - project_base_path (str): The base directory path of the project. + models_dir_path (str): The base directory path of the models. Attributes: - project_base_path (str): Stores the base directory path of the project. + models_dir_path (str): Stores the base directory path of the models. _files (Optional[str]): A list to store file paths, initialized as None. """ - self.project_base_path = project_base_path + self.models_dir_path = models_dir_path self._files: Optional[str] = None def get_files(self): - """Retrieves all files from the project base path. + """Retrieves all files from the models directory path. - This method walks through the directory structure starting from the project base path + This method walks through the directory structure starting from the models directory path and collects all file paths into the _files attribute. """ _files = [] # Walk through the directory structure and collect file paths - for root, _, fs in os.walk(self.project_base_path): + for root, _, fs in os.walk(self.models_dir_path): for file in fs: _files.append(os.path.join(root, file)) @@ -46,7 +46,7 @@ def get_files(self): @property def files(self): - """Returns the list of files in the project base path.""" + """Returns the list of files in the models directory path.""" if self._files is None: self.get_files() return self._files @@ -69,22 +69,22 @@ class ManifestLoader: """ Loads and parses manifest files for a data project. - This class is responsible for reading YAML manifest files from a given project base path, + This class is responsible for reading YAML manifest files from a given models directory path, parsing their contents, and populating a Manifest object with sources and other resources. """ - def __init__(self, project_base_path: str): + def __init__(self, models_dir_path: str): """ - Initializes the parser with the given project base path. + Initializes the parser with the given models directory path. Args: - project_base_path (str): The base directory path of the project. + models_dir_path (str): The base directory path of the models. Attributes: - project_base_path (str): Stores the base directory path of the project. + models_dir_path (str): Stores the base directory path of the models. manifest (Manifest): An instance of the Manifest class for managing project manifests. """ - self.project_base_path = project_base_path + self.models_dir_path = models_dir_path self.manifest = Manifest() def parse_source(self, srcs: dict): @@ -155,12 +155,12 @@ def parse_file_resources(self, data: dict): self.parse_resource(value, resource, resource_model) def load(self): - """Loads and parses all manifest files in the project base path.""" + """Loads and parses all manifest files in the models directory path.""" # get the file reader instance - file_reader = FileReaderFromFileSystem(self.project_base_path) + file_reader = FileReaderFromFileSystem(self.models_dir_path) - # get all yamls from the project base path + # get all yamls from the models directory path yaml_files = file_reader.filter_yaml_files() # iterate through each yaml file and read its contents and populate the manifest diff --git a/src/intugle/semantic_model.py b/src/intugle/semantic_model.py index abe6cee..dae6e5f 100644 --- a/src/intugle/semantic_model.py +++ b/src/intugle/semantic_model.py @@ -124,7 +124,7 @@ def export(self, format: str, **kwargs): from intugle.core import settings from intugle.parser.manifest import ManifestLoader - manifest_loader = ManifestLoader(settings.PROJECT_BASE) + manifest_loader = ManifestLoader(settings.MODELS_DIR) manifest_loader.load() manifest = manifest_loader.manifest @@ -212,7 +212,7 @@ def deploy(self, target: str, **kwargs): from intugle.core import settings from intugle.parser.manifest import ManifestLoader - manifest_loader = ManifestLoader(settings.PROJECT_BASE) + manifest_loader = ManifestLoader(settings.MODELS_DIR) manifest_loader.load() manifest = manifest_loader.manifest @@ -262,3 +262,4 @@ def deploy(self, target: str, **kwargs): f"Failed to deploy semantic model to '{target}': {e}", style="bold red" ) raise + diff --git a/src/intugle/semantic_search.py b/src/intugle/semantic_search.py index ee8b3a5..41e2c93 100644 --- a/src/intugle/semantic_search.py +++ b/src/intugle/semantic_search.py @@ -48,14 +48,14 @@ def thread_target(): class SemanticSearch: def __init__( - self, project_base: str = settings.PROJECT_BASE, collection_name: str = settings.VECTOR_COLLECTION_NAME + self, models_dir_path: str = settings.MODELS_DIR, collection_name: str = settings.VECTOR_COLLECTION_NAME ): - self.manifest_loader = ManifestLoader(project_base) + self.manifest_loader = ManifestLoader(models_dir_path) self.manifest_loader.load() self.manifest = self.manifest_loader.manifest self.collection_name = collection_name - self.project_base = project_base + self.models_dir_path = models_dir_path def get_column_details(self): sources = self.manifest.sources diff --git a/src/intugle/utils/files.py b/src/intugle/utils/files.py index beb0107..5fbb567 100644 --- a/src/intugle/utils/files.py +++ b/src/intugle/utils/files.py @@ -21,7 +21,7 @@ def update_relationship_file_mtime() -> None: if not settings.RELATIONSHIPS_FILE: return - file_path = os.path.join(settings.PROJECT_BASE, settings.RELATIONSHIPS_FILE) + file_path = os.path.join(settings.MODELS_DIR, settings.RELATIONSHIPS_FILE) # Check if the file exists before touching it if os.path.exists(file_path): diff --git a/tests/query_generator/test_query_generator.py b/tests/query_generator/test_query_generator.py index 648fa92..55173d5 100644 --- a/tests/query_generator/test_query_generator.py +++ b/tests/query_generator/test_query_generator.py @@ -139,7 +139,7 @@ def test_query_generator_is_isolated(mock_load, mock_manifest): mock_instance.manifest = mock_manifest # 3. Initialize DataProduct. The project_base path is now irrelevant. - sql_generator = DataProduct(project_base="/dummy/path") + sql_generator = DataProduct(models_dir_path="/dummy/path") generated_query = sql_generator.generate_query(etl_model) # 4. Define the expected query and assert the result diff --git a/tests/semantic_search/test_semantic_search.py b/tests/semantic_search/test_semantic_search.py index a05056b..18cc43e 100644 --- a/tests/semantic_search/test_semantic_search.py +++ b/tests/semantic_search/test_semantic_search.py @@ -174,7 +174,7 @@ def test_live_search_end_to_end(live_test_project_dir): """ # 1. Arrange: Instantiate SemanticSearch with the temporary project directory # This will use real clients and APIs because they are not mocked. - search_client = SemanticSearch(project_base=live_test_project_dir) + search_client = SemanticSearch(models_dir_path=live_test_project_dir) # 2. Act: Run the full initialize and search process try: From 8494d5b7078fa85c1d29be38dc936b4f84659595 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Thu, 23 Oct 2025 13:53:25 +0530 Subject: [PATCH 03/26] updated gitignore to include intugle project base folder --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 65f1326..ac0d473 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,5 @@ archived/ profiles.yml profiles.yaml + +/intugle/ From 3a95a72c65a9823df76357d746f8aada73fa64be Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Fri, 24 Oct 2025 17:08:00 +0530 Subject: [PATCH 04/26] column graph built, ruff linting --- src/intugle/adapters/types/duckdb/duckdb.py | 2 +- src/intugle/conceptual_search/models.py | 7 +- .../networkx_initializers.py | 117 +++++++++ .../graph_based_column_search/utils.py | 229 ++++++++++++++++++ src/intugle/core/conceptual_search/models.py | 30 +++ src/intugle/core/conceptual_search/utils.py | 147 +++++++++++ .../core/pipeline/link_prediction/lp.py | 2 +- src/intugle/core/settings.py | 4 + src/intugle/core/vector_store/__init__.py | 5 +- src/intugle/core/vector_store/qdrant.py | 18 +- src/intugle/mcp/semantic_layer/prompt.py | 4 +- streamlit_app/helper.py | 26 +- streamlit_app/main.py | 137 +++++------ 13 files changed, 621 insertions(+), 107 deletions(-) create mode 100644 src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py create mode 100644 src/intugle/core/conceptual_search/graph_based_column_search/utils.py create mode 100644 src/intugle/core/conceptual_search/models.py create mode 100644 src/intugle/core/conceptual_search/utils.py diff --git a/src/intugle/adapters/types/duckdb/duckdb.py b/src/intugle/adapters/types/duckdb/duckdb.py index 54591be..2ffa9ea 100644 --- a/src/intugle/adapters/types/duckdb/duckdb.py +++ b/src/intugle/adapters/types/duckdb/duckdb.py @@ -1,5 +1,5 @@ import time -import re + from typing import TYPE_CHECKING, Any, Optional import duckdb diff --git a/src/intugle/conceptual_search/models.py b/src/intugle/conceptual_search/models.py index d842581..67d835e 100644 --- a/src/intugle/conceptual_search/models.py +++ b/src/intugle/conceptual_search/models.py @@ -1,13 +1,16 @@ # src/intugle/conceptual_search/models.py -from typing import Optional, List +from typing import List, Optional + from pydantic import BaseModel, Field + class ConceptualAttribute(BaseModel): """An abstract attribute for the desired data product.""" name: str = Field(description="A business-friendly name for the column, e.g., 'Patient Age'") description: str = Field(description="A detailed explanation of what this attribute represents.") logic: str = Field(description="A natural language description of the calculation, e.g., 'Calculate the age based on the date of birth'") + class ConceptualPlan(BaseModel): """The conceptual plan generated by Agent 1, which can be reviewed by the user.""" attributes: List[ConceptualAttribute] @@ -15,11 +18,13 @@ class ConceptualPlan(BaseModel): def to_dict(self): return self.model_dump() + class MappedAttribute(ConceptualAttribute): """A conceptual attribute that has been mapped to a physical column.""" column_id: str = Field(description="The fully qualified column ID from the semantic layer, e.g., 'patients.birth_date'") measure_func: Optional[str] = Field(default=None, description="The aggregation function if any, e.g., 'COUNT', 'SUM'") + class MappedPlan(BaseModel): """The final plan after Agent 2 has mapped attributes to columns.""" attributes: List[MappedAttribute] diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py new file mode 100644 index 0000000..6510f4b --- /dev/null +++ b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py @@ -0,0 +1,117 @@ +import asyncio +import logging +import os +import pickle + +import networkx as nx + +from intugle.core import settings +from intugle.core.conceptual_search.graph_based_column_search.utils import ( + create_embeddings, + prepare_chunk_document, +) +from intugle.core.conceptual_search.models import GraphFileName +from intugle.core.conceptual_search.utils import colbert_score_numpy + +log = logging.getLogger(__name__) + + +def build_knowledge_graph(doc): + """ + Build a knowledge graph from text chunks. + + Args: + chunks (List[Dict]): List of text chunks with metadata + model (str): Embedding model name + + Returns: + Tuple[nx.Graph, List[np.ndarray]]: The knowledge graph and chunk embeddings + """ + log.info("Building knowledge graph...") + + # Create a graph + graph = nx.Graph() + + # Create embeddings for all chunks + log.info("Creating embeddings for chunks...") + print("*" * 100) + log.info(f"doc length: {len(doc)}") + print("*" * 100) + + embeddings = asyncio.run(create_embeddings(doc)) + + # Add nodes to the graph + print("Adding nodes to the graph...") + for i, chunk in enumerate(doc): + + # concept_i = chunk.metadata['concepts'] + # Add node with attributes + graph.add_node(i, source=chunk.metadata['source']) + # concepts=concept_i,embedding=embeddings[i]) + log.info(f"node {i} added ") + + # Connect nodes based on shared concepts + # embedding_model = PreloadedEmbedding.bge_m3_model + + log.info("Creating edges between nodes...") + weights = [] + for i in range(len(doc)): + # node_concepts = set(graph.nodes[i]["concepts"]) + concepts_i = doc[i].metadata["concepts"] + node_concepts = set(concepts_i) + + for j in range(i + 1, len(doc)): + # Calculate concept overlap + # other_concepts = set(graph.nodes[j]["concepts"]) + other_concepts = set(doc[j].metadata['concepts']) + shared_concepts = node_concepts.intersection(other_concepts) + + # If they share concepts, add an edge + if shared_concepts: + + similarity = colbert_score_numpy(embeddings[i], embeddings[j]) + + # Calculate edge weight based on concept overlap and semantic similarity + concept_score = len(shared_concepts) / min(len(node_concepts), len(other_concepts)) + + edge_weight = 0.7 * similarity + 0.3 * concept_score + + weights.append(edge_weight) + + # Only add edges with significant relationship + if edge_weight > 0.95: + graph.add_edge(i, j, + weight=edge_weight, + similarity=similarity, + ) + # shared_concepts=list(shared_concepts)) + + log.info(f"Knowledge graph built with {graph.number_of_nodes()} nodes and {graph.number_of_edges()} edges") + + return graph, embeddings + + +def prepare_networkx_graph(manifest): + + graph_path = os.path.join(settings.GRAPH_DIR, + GraphFileName.FIELD) + + if os.path.exists(graph_path): + print('[!] Column search graph already build ... skipping the step') + return + + docs = prepare_chunk_document(manifest) + + graph, _ = build_knowledge_graph(doc=docs) + + os.makedirs(os.path.join(settings.GRAPH_DIR), exist_ok=True) + + # write as pickle + try: + with open(graph_path, "wb") as _file: + pickle.dump(graph, _file, pickle.HIGHEST_PROTOCOL) + except Exception as ex: + raise ex + + print(f"Graph saved to {graph_path}") + print(f"Graph created with {graph.number_of_nodes()} nodes and {graph.number_of_edges()} edges") \ No newline at end of file diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py new file mode 100644 index 0000000..eb34360 --- /dev/null +++ b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py @@ -0,0 +1,229 @@ +import ast +import logging +import os + +from itertools import chain +from pathlib import Path +from uuid import uuid4 + +import numpy as np +import pandas as pd + +from qdrant_client import models +from tqdm import tqdm + +from intugle.core import settings +from intugle.core.conceptual_search.models import QdrantCollectionSuffix +from intugle.core.conceptual_search.utils import ( + batched, + extract_concepts_column, + fetch_column_with_description, +) +from intugle.core.llms.chat import ChatModelLLM +from intugle.core.llms.embeddings import Embeddings, EmbeddingsType +from intugle.core.vector_store import AsyncQdrantService, VDocument +from intugle.core.vector_store.qdrant import QdrantVectorConfiguration + +log = logging.getLogger(__name__) + +tqdm.pandas() + +CONCEPTUAL_SEARCH_COLLECTION_NAME = "conceptual_search_columns" + + +def collection_name(prefix: str, suffix: str = QdrantCollectionSuffix.FIELD): + + return f"{prefix}{suffix}" + + +async def fetch_embeddings_from_qdrant(collection_name): + + data = pd.DataFrame() + + async with AsyncQdrantService(collection_name=collection_name, collection_configurations={}) as qdb: + + data = await qdb.get(includes=["metadata", "embeddings"], return_document=False) + + data = pd.DataFrame([d.model_dump() for d in data])[["payload", "vector"]] + + if data.shape[0] > 0: + + data = pd.concat( + [data, pd.json_normalize(data.payload)], axis=1 + )[["row", "source", "vector", "content"]] # [["row", "source", "concepts","vector","content"]] + + if "row" in data.columns: + + return data.sort_values(by="row", ascending=True).reset_index(drop=True) + + return data + + +async def create_embeddings(doc: list[VDocument], recreate: bool = False): + """ + Create and store embeddings for the given documents using the configured embedding model. + This function aligns with the intugle architecture for handling embeddings and vector stores. + """ + # 1. Initialize the standard Embeddings class from project settings + embedding_model = Embeddings( + model_name=settings.EMBEDDING_MODEL_NAME, + tokenizer_model=settings.TOKENIZER_MODEL_NAME, + ) + + # 2. Create the vector store configuration programmatically + vectors_config = { + f"{embedding_model.model_name}-{EmbeddingsType.LATE}": models.VectorParams( + size=embedding_model.embeddings_size, + distance=models.Distance.COSINE, + multivector_config=models.MultiVectorConfig( + comparator=models.MultiVectorComparator.MAX_SIM + ), + ), + } + configuration = QdrantVectorConfiguration(vectors_config=vectors_config) + + # 3. Instantiate the AsyncQdrantService + async with AsyncQdrantService( + collection_name=CONCEPTUAL_SEARCH_COLLECTION_NAME, + collection_configurations=configuration, + ) as qdb: + if recreate: + await qdb.delete_collection() + await qdb.create_collection() + log.info(f"[*] Deleted and recreated collection {CONCEPTUAL_SEARCH_COLLECTION_NAME}") + + # 4. Check if documents need to be vectorized and inserted + count_result = await qdb.count() + count_req = count_result.count + if count_req < len(doc): + log.info("Vectorizing and adding documents to Qdrant...") + # 5. Vectorize documents + vectors = await embedding_model.aencode( + [d.page_content for d in doc], + embeddings_types=[EmbeddingsType.LATE], + ) + late_vectors = vectors[EmbeddingsType.LATE] + + # 6. Create PointStruct objects for insertion + points = [] + for i, document in enumerate(doc): + point_id = str(uuid4()) + vector = {f"{embedding_model.model_name}-{EmbeddingsType.LATE}": late_vectors[i]} + # Merge metadata and page_content into the payload + payload = {**document.metadata, "content": document.page_content} + points.append( + models.PointStruct( + id=point_id, + vector=vector, + payload=payload, + ) + ) + + # 7. Bulk insert into Qdrant + for batch in batched(points, 30): + qdb.bulk_insert(points=batch) + log.info("Embeddings created and inserted into Qdrant DB") + + # 8. Fetch and return the embeddings + log.info("Fetching embeddings from Qdrant...") + all_vector_records = await fetch_embeddings_from_qdrant( + collection_name=CONCEPTUAL_SEARCH_COLLECTION_NAME + ) + log.info("Embeddings fetched from Qdrant.") + + vector_name = f"{embedding_model.model_name}-{EmbeddingsType.LATE}" + all_embeddings = [ + np.array(item["vector"][vector_name]) + for _, item in all_vector_records.iterrows() + ] + + return all_embeddings + + +def filter_out_concepts(data: pd.DataFrame, threshold: float = 0.1): + total_documents = data.shape[0] + concepts = pd.DataFrame(list(chain(*data.concepts.tolist())), columns=["concepts"]) + value_counts = concepts.value_counts().reset_index() + value_counts["proportion"] = value_counts["count"] / total_documents + + return value_counts.loc[value_counts.proportion > threshold].concepts.tolist() + + +def clean_concepts(concepts: list): + + if isinstance(concepts, list): + + return list(map(lambda concept: concept.strip().lower(), concepts)) + + return concepts + + +def remove_unwanted_concepts(concepts: list, filter_out: list): + + return list(filter(lambda concept: concept not in filter_out, concepts)) + + +def prepare_chunk_document(manifest): + column_description_path = os.path.join(settings.COL_DESCRIPTION_DIR, "column_description.csv") + column_description_path = Path(column_description_path) + column_description_path.parent.mkdir(exist_ok=True, parents=True) + + llm = ChatModelLLM.get_llm( + model_name=settings.LLM_PROVIDER, + llm_config={"temperature": 0.05}, + ) + if not column_description_path.exists(): + + # existing = pd.read_csv(column_description_path) + + column_description_df = fetch_column_with_description(manifest) + + column_description_df['Text'] = column_description_df['column_name'] + " - " + column_description_df['business_glossary'] + + column_description_df['id'] = column_description_df['table_name'] + "$$##$$" + column_description_df['column_name'] + + # remaining_columns = set(column_description_df["id"].unique()) - set(existing["id"].unique()) + + # column_description_df = column_description_df[(column_description_df["business_glossary"] != "") & (column_description_df["id"].isin(list(remaining_columns)))].reset_index(drop=True) + + column_description_df = column_description_df[(column_description_df["business_glossary"] != "")].reset_index(drop=True) + + log.info(f"[*] Extracting concepts for {column_description_df.shape[0]} columns") + + for i, data in tqdm(enumerate(batched(column_description_df, 30), 1), position=0, leave=True): + + column_description_df.loc[data.index, "concepts"] = data["Text"].progress_apply(lambda text: clean_concepts(extract_concepts_column(text=text, llm=llm))) + + # column_description_df.loc[data.index,"concepts"] = data["business_tags"].apply(clean_concepts) + + column_description_df.loc[data.index, "concepts"] = column_description_df.loc[data.index].apply(lambda row: remove_unwanted_concepts(concepts=row["concepts"], filter_out=[row['column_name'], 'sap erp', 'sap erp system', 'sap']), axis=1) + + # breakpoint() + + column_description_df.loc[data.index].to_csv(column_description_path, + index=False, + mode='a', + header=not os.path.exists(column_description_path)) + + log.info(f"[*] Batch {i} completed ... sleeping for 60 seconds") + + # time.sleep(60) + + else: + + column_description_df = pd.read_csv(column_description_path) + + column_description_df["concepts"] = column_description_df.concepts.apply(ast.literal_eval) + + concepts_to_be_removed = filter_out_concepts(data=column_description_df[["concepts"]]) + + # ["unique identifier","transaction","transaction code"] + column_description_df["concepts"] = column_description_df.apply(lambda row: remove_unwanted_concepts(concepts=row["concepts"], filter_out=concepts_to_be_removed), axis=1) + + data_documents = [] + + for index, row in column_description_df.iterrows(): + + data_documents.append(VDocument(page_content=row['Text'].strip().lower(), metadata={"source": row['id'], "row": index, "concepts": row['concepts']})) + + return data_documents \ No newline at end of file diff --git a/src/intugle/core/conceptual_search/models.py b/src/intugle/core/conceptual_search/models.py new file mode 100644 index 0000000..db72dbf --- /dev/null +++ b/src/intugle/core/conceptual_search/models.py @@ -0,0 +1,30 @@ +from enum import Enum + +from pydantic import BaseModel, Field + + +class Relevance_Classification(BaseModel): + relevance_score: int = Field( + description="describes how relevant the column is, the higher the number the more relevant", + enum=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + ) + + +class QdrantCollectionSuffix(str, Enum): + TABLE = "_table_shortlisting" + FIELD = "_field_shortlisting" + + def __repr__( + self, + ): + return str(self.value) + + +class GraphFileName(str, Enum): + TABLE = "table_graph.gpickle" + FIELD = "field_graph.gpickle" + + def __repr__( + self, + ): + return str(self.value) diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py new file mode 100644 index 0000000..bcd2e1a --- /dev/null +++ b/src/intugle/core/conceptual_search/utils.py @@ -0,0 +1,147 @@ +import json +import logging +import re + +from typing import TYPE_CHECKING, Any + +import numpy as np +import pandas as pd + +if TYPE_CHECKING: + from intugle.parser.manifest import Manifest + +log = logging.getLogger(__name__) + + +def batched(data: Any, n): + for index in range(0, len(data), n): + yield data[index : index + n] + + +def colbert_score_numpy( + query_embeddings: np.ndarray, doc_embeddings: np.ndarray +) -> float: + """ + ColBERT score using NumPy. + + Args: + query_embeddings: np.ndarray of shape (q_len, dim) + doc_embeddings: np.ndarray of shape (d_len, dim) + + Returns: + float: ColBERT relevance score + """ + # Normalize + query_norm = query_embeddings / np.linalg.norm( + query_embeddings, axis=1, keepdims=True + ) + doc_norm = doc_embeddings / np.linalg.norm(doc_embeddings, axis=1, keepdims=True) + + # Dot product matrix (q_len x d_len) + similarity_matrix = np.dot(query_norm, doc_norm.T) + + # MaxSim for each query token + max_similarities = np.max(similarity_matrix, axis=1) + + # Sum of max similarities + score = np.sum(max_similarities) + + return float(score) / len(query_embeddings) + + +def manual_concept_extraction(ai_msg): + try: + # Try to parse a clean JSON array of strings + concepts = json.loads(ai_msg.content) + if isinstance(concepts, list) and all(isinstance(c, str) for c in concepts): + return concepts + else: + log.info("Warning: JSON was parsed but did not return a list of strings.") + except (json.JSONDecodeError, AttributeError, TypeError) as e: + log.info(f"Primary JSON parsing failed: {e}") + + # --- Fallback strategy --- + try: + raw_text = getattr(ai_msg, "content", "") + # Attempt to extract a list manually from the raw text + matches = re.findall(r"\[(.*?)\]", raw_text, re.DOTALL) + if matches: + items = re.findall(r'"([^"]+)"', matches[0]) + return items + else: + log.info("Fallback regex parsing did not find any matches.") + except Exception as ex: + log.error(f"Fallback regex parsing didnot find any matches\nReason: {ex}") + + return [] + + +def extract_concepts_column(text, llm): + """ + Extract key concepts from text using GROQ's API. + + Args: + text (str): Text to extract concepts from + + Returns: + List[str]: List of key concepts or entities + """ + system_message = """ + # Instructions: + - Extract key concepts and entities from the provided description of a field in database . + - Return ONLY a list of 2 key terms, entities, or concepts that are most important in this text. + - Return a valid JSON array of strings. Example:["Entity1", "Entity2"] + - Donot mention table names or field name itself, or irrelevant terms , irrelevant entities or irreleavant concepts. + - The list of terms should be unique. + """ + + messages = [ + ("system", system_message), + ("user", f"Extract key concepts from field description:\n\n{text[:3000]}"), + ] + try: + ai_msg = llm.invoke(messages) + concepts = manual_concept_extraction(ai_msg) + return concepts + except Exception as fallback_err: + log.info(f"Fallback parsing also failed: {fallback_err}") + + return [] + + +def fetch_column_with_description(manifest: "Manifest") -> pd.DataFrame: + """ + Fetches all column details from the manifest. + This replaces the original SQL-based implementation. + """ + column_data = [] + for source in manifest.sources.values(): + table = source.table + if not table or not table.columns: + continue + + for column in table.columns: + column_data.append( + { + "id": f"{table.name}.{column.name}", + "table_name": table.name, + "column_name": column.name, + "business_glossary": column.description or "", + "business_tags": column.tags or [], + "db_schema": source.schema, + } + ) + + if not column_data: + return pd.DataFrame( + columns=[ + "id", + "table_name", + "column_name", + "business_glossary", + "business_tags", + "db_schema", + ] + ) + + return pd.DataFrame(column_data) \ No newline at end of file diff --git a/src/intugle/core/pipeline/link_prediction/lp.py b/src/intugle/core/pipeline/link_prediction/lp.py index 63a26d5..3ccd308 100644 --- a/src/intugle/core/pipeline/link_prediction/lp.py +++ b/src/intugle/core/pipeline/link_prediction/lp.py @@ -1,6 +1,6 @@ +import ast import logging import time -import ast from enum import Enum from typing import List, Optional, Tuple diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index 4b2f789..d511442 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -39,9 +39,13 @@ class Settings(BaseSettings): DI_MODEL_VERSION: str = "13052023" + # DIRECTORY STRUCTURE ENVS PROJECT_BASE: str = create_project_base_if_not_exists() MODELS_DIR_NAME: str = "models" + GRAPH_DIR_NAME: str = "kg" MODELS_DIR: str = os.path.join(PROJECT_BASE, MODELS_DIR_NAME) + GRAPH_DIR: str = os.path.join(PROJECT_BASE, GRAPH_DIR_NAME) + COL_DESCRIPTION_DIR: str = os.path.join(PROJECT_BASE, "column_descriptions") PROFILES_PATH: str = os.path.join(os.getcwd(), "profiles.yml") diff --git a/src/intugle/core/vector_store/__init__.py b/src/intugle/core/vector_store/__init__.py index a13b90f..28d0b24 100644 --- a/src/intugle/core/vector_store/__init__.py +++ b/src/intugle/core/vector_store/__init__.py @@ -1,3 +1,4 @@ -from intugle.core.vector_store.qdrant import AsyncQdrantService +from intugle.core.vector_store.qdrant import AsyncQdrantService, VDocument -VectorStoreService = AsyncQdrantService \ No newline at end of file +VectorStoreService = AsyncQdrantService +VDocument = VDocument \ No newline at end of file diff --git a/src/intugle/core/vector_store/qdrant.py b/src/intugle/core/vector_store/qdrant.py index 64afefa..f83301e 100644 --- a/src/intugle/core/vector_store/qdrant.py +++ b/src/intugle/core/vector_store/qdrant.py @@ -1,4 +1,3 @@ -import asyncio import logging import sys @@ -17,17 +16,25 @@ class QdrantVectorConfiguration(BaseModel): + vectors_config: Optional[qdrant_types.VectorParams | Mapping[str, qdrant_types.VectorParams]] = None + sparse_vectors_config: Optional[Mapping[str, qdrant_types.SparseVectorParams]] = None # Used for standardization + class VDocument(BaseModel): + id: Optional[str | int] = None + page_content: Optional[str] = None + metadata: Optional[Dict[str, Any]] = None + options: Dict[str, Any] = {} # Refere to QDocumentOptions - embeddings: Optional[Dict[str, List[float]]] = None + + embeddings: Optional[Dict[str, List[float] | List[List[float]]]] = None # Used for standardization @@ -119,6 +126,13 @@ async def create_collection( log.error(f"AsyncQdrantService: Couldn't create collection, reason: {e}") raise e + async def count(self) -> qdrant_types.CountResult: + try: + return await self.client.count(collection_name=self.collection_name) + except Exception as e: + log.error(f"AsyncQdrantService: Couldn't count collection, reason: {e}") + raise e + def bulk_insert(self, points: models.PointStruct | List[models.PointStruct]): try: result = self.upload_point(points) diff --git a/src/intugle/mcp/semantic_layer/prompt.py b/src/intugle/mcp/semantic_layer/prompt.py index c42ff27..7c237a2 100644 --- a/src/intugle/mcp/semantic_layer/prompt.py +++ b/src/intugle/mcp/semantic_layer/prompt.py @@ -1,7 +1,9 @@ import textwrap -import aiofiles + from pathlib import Path +import aiofiles + from intugle.mcp.docs_search.service import docs_search_service from intugle.mcp.semantic_layer.schema import SQLDialect diff --git a/streamlit_app/helper.py b/streamlit_app/helper.py index aae57da..f2513c7 100644 --- a/streamlit_app/helper.py +++ b/streamlit_app/helper.py @@ -3,31 +3,20 @@ # Import Packages # ----------------------- # Standard library -import hashlib import io import os import re -import shutil -import time -import uuid import zipfile + from dataclasses import asdict, is_dataclass -from datetime import datetime from pathlib import Path -from typing import Dict, Mapping, Optional, Tuple, Literal -from typing import Any, Iterable, Mapping, Sequence -from urllib.parse import unquote, urlparse +from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, Tuple # Third-party -import matplotlib.pyplot as plt import networkx as nx import pandas as pd import plotly.graph_objects as go -import requests import streamlit as st -from graphviz import Digraph - - # ----------------------- # Helpers @@ -104,7 +93,7 @@ # # --- open dialog on first load # open_email_dialog() -def build_yaml_zip(asset_dir: str ) -> tuple[bytes, int]: +def build_yaml_zip(asset_dir: str) -> tuple[bytes, int]: """ Create an in-memory ZIP archive containing all .yml/.yaml files under `asset_dir`. @@ -532,6 +521,7 @@ def normalize(col: str) -> str: # ------------------------------ File β†’ DataFrame ------------------------------ + def read_file_to_df(uploaded_file) -> Tuple[pd.DataFrame, str]: """ Read an uploaded CSV/XLS/XLSX into a DataFrame. @@ -706,6 +696,7 @@ def _get_secret_env(name: str) -> str | None: # ------------------------------ Name helpers ------------------------------ + def _normalized_table_name(raw: str) -> str: """ Normalize table names using the strongest available normalizer. @@ -716,7 +707,7 @@ def _normalized_table_name(raw: str) -> str: return normalize_col_name(raw) # type: ignore[name-defined] -def _csv_path_for(name: str,MODIFIED_DIR) -> Path: +def _csv_path_for(name: str, MODIFIED_DIR) -> Path: """ Return the output CSV path for a given table name within MODIFIED_DIR. Requires `MODIFIED_DIR` and `safe_filename` to be defined globally. @@ -836,6 +827,7 @@ def predicted_links_to_df(links: Sequence[Any]) -> pd.DataFrame: # ---------- new: plotly network (tables as nodes) ---------- + def plotly_table_graph( df: pd.DataFrame, *, @@ -966,7 +958,7 @@ def edge_hover(u: str, v: str, data: Mapping[str, Any]) -> str: if len(data["labels"]) == 1: edge_label_text.append(data["labels"][0]) else: - edge_label_text.append(f"{data['labels'][0]} (+{len(data['labels'])-1} more)") + edge_label_text.append(f"{data['labels'][0]} (+{len(data['labels']) - 1} more)") edge_traces.append( go.Scatter( @@ -1004,7 +996,7 @@ def edge_hover(u: str, v: str, data: Mapping[str, Any]) -> str: if len(data["labels"]) == 1: edge_label_text.append(data["labels"][0]) else: - edge_label_text.append(f"{data['labels'][0]} (+{len(data['labels'])-1} more)") + edge_label_text.append(f"{data['labels'][0]} (+{len(data['labels']) - 1} more)") # Edge labels as a single text layer edge_label_trace = go.Scatter( diff --git a/streamlit_app/main.py b/streamlit_app/main.py index 358f42f..67edd74 100644 --- a/streamlit_app/main.py +++ b/streamlit_app/main.py @@ -4,30 +4,22 @@ # ----------------------- # Standard library import hashlib -import io import os import re import shutil import time -import uuid -import zipfile -from dataclasses import asdict, is_dataclass + from datetime import datetime from pathlib import Path -from typing import Dict, Mapping, Optional, Tuple, Literal -from urllib.parse import unquote, urlparse +from typing import Dict, Literal # Third-party -import matplotlib.pyplot as plt -import networkx as nx import pandas as pd -import plotly.graph_objects as go -import requests import streamlit as st + from graphviz import Digraph -from langchain_openai import ChatOpenAI from langchain_google_genai import ChatGoogleGenerativeAI -from langchain_openai import AzureChatOpenAI +from langchain_openai import AzureChatOpenAI, ChatOpenAI # Local package: intugle try: @@ -38,7 +30,7 @@ st.error("`intugle` package not available. Please install and restart the app.") st.stop() from helper import * -from helper import _on_select_change, _on_name_edit, _csv_path_for, _normalized_table_name +from helper import _csv_path_for, _normalized_table_name, _on_name_edit, _on_select_change # ----------------------- # Config & constants @@ -110,11 +102,13 @@ ACCEPTED_TYPES = (".csv", ".xls", ".xlsx") PREVIEW_ROWS: int = 100 + # ---- Session-scoped base directory helpers ---- def _timestamp_folder() -> str: """Return a filesystem-safe timestamp like '2025-10-19_14-32-07'.""" return datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + def ensure_base_dir() -> Path: """ Create and store a per-session base directory exactly once. @@ -130,11 +124,13 @@ def ensure_base_dir() -> Path: st.toast(f"Base dir: {base}") return base + def relpath(path: str | Path) -> Path: """Build a path under the session's base dir.""" base = ensure_base_dir() return base / Path(path) + # ---- Initialize I/O directories ---- def init_io_dirs() -> Dict[str, Path]: """ @@ -160,6 +156,7 @@ def init_io_dirs() -> Dict[str, Path]: return paths + # ---- Call once near top of script ---- dirs = init_io_dirs() INPUT_DIR = dirs["INPUT_DIR"] @@ -195,11 +192,13 @@ def init_io_dirs() -> Dict[str, Path]: "just_uploaded_rerun": False, } + def init_session_state(defaults: dict) -> None: """Idempotently seed st.session_state with defaults.""" for k, v in defaults.items(): st.session_state.setdefault(k, v) + # Initialize once at top of file init_session_state(DEFAULT_STATE) @@ -264,13 +263,11 @@ def init_session_state(defaults: dict) -> None: # upload_expander = False - - # ----------------------- # UI: Header # ----------------------- -### download image +# download image # if upload_expander: # # url = "https://commons.wikimedia.org/wiki/File:Intugle_icon.png" # url = "https://commons.wikimedia.org/wiki/File:Intugle_main_logo.png" @@ -278,7 +275,7 @@ def init_session_state(defaults: dict) -> None: # print(f"Image downloaded to: {path}") # Layout: logo + title side by side -_ , tcol1, tcol2 = st.columns([1, 2, 0.5]) +_, tcol1, tcol2 = st.columns([1, 2, 0.5]) with tcol1: st.title(":violet[Intugle Semantic Modeler]") @@ -291,29 +288,27 @@ def init_session_state(defaults: dict) -> None: st.logo( # 'https://commons.wikimedia.org/wiki/File:Intugle_icon.png', "intugle_assets/Intugle_main_logo.png", - size = 'large' + size='large' ) - - # ========================= # Status Bar (single row) # ========================= # ---- your booleans -upload_done = bool(st.session_state["seen_files"]) -creds_done = bool(st.session_state["creds_saved"]) -freeze_done = st.session_state.get("route") == "semantic_all" +upload_done = bool(st.session_state["seen_files"]) +creds_done = bool(st.session_state["creds_saved"]) +freeze_done = st.session_state.get("route") == "semantic_all" profiling_done = bool(st.session_state["data_profiling_glossary"]) -links_done = bool(st.session_state["semantic_model_done"]) +links_done = bool(st.session_state["semantic_model_done"]) # ---- stages in order stages = [ - ("Upload Files", upload_done), - ("LLM Credentials", creds_done), - ("Rename Tables & Columns", freeze_done), - ("Profiling & Glossary", profiling_done), + ("Upload Files", upload_done), + ("LLM Credentials", creds_done), + ("Rename Tables & Columns", freeze_done), + ("Profiling & Glossary", profiling_done), ("Semantic Link Identification", links_done), ] @@ -363,7 +358,7 @@ def init_session_state(defaults: dict) -> None: st.markdown("".join(html_parts) + "", unsafe_allow_html=True) st.divider() -#------------------------------------------ +# ------------------------------------------ # st.title("Intugle Semantic Modeler") @@ -421,7 +416,6 @@ def init_session_state(defaults: dict) -> None: accept_multiple_files=True, key=f"uploader_{st.session_state['uploader_key']}",) - if uploaded_files: st.caption(f"Received {len(uploaded_files)} file(s).") @@ -470,7 +464,7 @@ def init_session_state(defaults: dict) -> None: df = standardize_columns(df) # # Initial table name (no extra counters unless a different file collides) - input_path = INPUT_DIR / safe_filename(clean_table_name(upl.name), '.csv' ) + input_path = INPUT_DIR / safe_filename(clean_table_name(upl.name), '.csv') try: if (not input_path.exists()) or (hashlib.sha1(input_path.read_bytes()).hexdigest() != sha): input_path.write_bytes(file_bytes) @@ -506,7 +500,6 @@ def init_session_state(defaults: dict) -> None: "modified_path": str(modified_path), # <β€” new } - # Save table and fingerprint with st.spinner(f"Saving `{unique_default}`..."): add_table_to_state(unique_default, df, meta) @@ -532,7 +525,7 @@ def init_session_state(defaults: dict) -> None: st.toast("Processing completed for all eligible files.") # Clear the uploader selection and force exactly one rerun st.session_state["uploader_key"] += 1 # resets file_uploader - st.session_state["just_uploaded_rerun"] = True # one-shot marker + st.session_state["just_uploaded_rerun"] = True # one-shot marker st.rerun() # ----------------------- @@ -545,11 +538,9 @@ def init_session_state(defaults: dict) -> None: if row.get("source_file") == info.get("source_file") and row.get("status", "").startswith("βœ…"): row["table_name"] = latest_name - if st.session_state.get("tables") or st.session_state.get("ingest_log"): with st.expander("1) Review Tables", expanded=False): - # Summary now shows status/details only; no inline actions if st.session_state.get("ingest_log"): summary_df = pd.DataFrame(st.session_state["ingest_log"]).copy() @@ -578,7 +569,7 @@ def init_session_state(defaults: dict) -> None: st.toast(""" **Welcome to Intugle!** We’re thrilled to have you here. - """,icon='πŸ•ΈοΈ') + """, icon='πŸ•ΈοΈ') # ----------------------- # Modify table & columns (MAIN PAGE, under the review table) @@ -603,7 +594,7 @@ def init_session_state(defaults: dict) -> None: has_tables = bool(st.session_state.get("tables")) tnames = sorted(st.session_state["tables"].keys()) if has_tables else [] options = tnames if has_tables else ["β€” no tables loaded β€”"] - col1,col2,col3,col4 = st.columns([2,0.5,2,2]) + col1, col2, col3, col4 = st.columns([2, 0.5, 2, 2]) with col1: sel = st.selectbox( "Select table to modify", @@ -632,7 +623,7 @@ def init_session_state(defaults: dict) -> None: # st.markdown("
➑️
", unsafe_allow_html=True) st.image( "https://upload.wikimedia.org/wikipedia/commons/c/ca/Eo_circle_purple_arrow-right.svg", - width=80, # Manually Adjust the width of the image as per requirement + width=80, # Manually Adjust the width of the image as per requirement ) with col3: @@ -642,20 +633,19 @@ def init_session_state(defaults: dict) -> None: "Rename table (optional)", # key="main_modify_final_name", # value=st.session_state.get("main_modify_final_name", sel) or sel, - value = sel, + value=sel, on_change=_on_name_edit, help="Defaults to the selected table name. Will be normalized to lowercase with underscores." ) - # Current paths current_name = sel - current_path = _csv_path_for(current_name,MODIFIED_DIR) + current_path = _csv_path_for(current_name, MODIFIED_DIR) # ------------------------------- # ACTIONS # ------------------------------- - __,_, act_col1, ___, = st.columns([2,0.5,2,2]) + __, _, act_col1, ___, = st.columns([2, 0.5, 2, 2]) # 1) RENAME TABLE (and CSV file) β€” no column changes here with act_col1: @@ -668,7 +658,7 @@ def init_session_state(defaults: dict) -> None: if new_name == current_name: st.toast("Table name unchanged.") else: - new_path = _csv_path_for(new_name,MODIFIED_DIR) + new_path = _csv_path_for(new_name, MODIFIED_DIR) # Move/rename the CSV on disk (if it exists). If it doesn't, create from in-memory df. try: @@ -704,8 +694,6 @@ def init_session_state(defaults: dict) -> None: st.success(f"Renamed **{current_name}** β†’ **{new_name}** and updated CSV file.") st.rerun() - - # Column editor (unchanged) st.markdown("**Edit or ignore columns**") editor_src = { @@ -727,13 +715,12 @@ def init_session_state(defaults: dict) -> None: # ---------- Editor UI already created above ---------- # sel, df_current, orig_cols, final_name_raw, edit_df exist - # 2) FREEZE COLUMN NAMES β€” apply edits/ignores & save back to SAME file name (CSV overwrite) # with act_col2: if st.button("Freeze column names", type="primary", width="content", key="btn_freeze_cols"): # Pull edits new_names_input = list(edit_df["new_column_name"]) - ignored_flags = list(edit_df["ignore_column"]) + ignored_flags = list(edit_df["ignore_column"]) # Validate proposed names errs = validate_column_names(orig_cols, new_names_input, ignored_flags) @@ -743,9 +730,9 @@ def init_session_state(defaults: dict) -> None: st.stop() # Build modified DataFrame - keep_idx = [i for i, ig in enumerate(ignored_flags) if not ig] - kept_cols = [orig_cols[i] for i in keep_idx] - new_names = [normalize_col_name(new_names_input[i]) for i in keep_idx] + keep_idx = [i for i, ig in enumerate(ignored_flags) if not ig] + kept_cols = [orig_cols[i] for i in keep_idx] + new_names = [normalize_col_name(new_names_input[i]) for i in keep_idx] df_mod = df_current.loc[:, kept_cols].copy() df_mod.columns = new_names @@ -754,7 +741,7 @@ def init_session_state(defaults: dict) -> None: # live_name = st.session_state["main_modify_select"] live_name = final_name_raw st.info(live_name) - live_path = _csv_path_for(live_name,MODIFIED_DIR) + live_path = _csv_path_for(live_name, MODIFIED_DIR) # Save back to the SAME file (overwrite) try: @@ -773,9 +760,6 @@ def init_session_state(defaults: dict) -> None: st.rerun() - - - # ----------------------- # Sidebar # ----------------------- @@ -795,8 +779,6 @@ def init_session_state(defaults: dict) -> None: # st.write(f"**OPENAI_API_KEY:** {mask(get_secret('OPENAI_API_KEY'))}") # pip install -U langchain-openai - - llm_temp = ChatOpenAI( model=os.environ["LLM_PROVIDER"], # pick your OpenAI model temperature=0, @@ -838,15 +820,13 @@ def init_session_state(defaults: dict) -> None: # st.write(f'OPENAI_API_VERSION: {os.environ["OPENAI_API_VERSION"]}') # st.write(f'AZURE_OPENAI_ENDPOINT: {os.environ["AZURE_OPENAI_ENDPOINT"]}') - elif choice == "gemini": # st.write(f"**Model:** `{cfg.get('model','')}`") # Common env names for Gemini: GEMINI_API_KEY or GOOGLE_API_KEY # st.write(f"**GEMINI/GOOGLE_API_KEY:** {mask(get_secret('GEMINI_API_KEY') or get_secret('GOOGLE_API_KEY'))}") - llm_temp = ChatGoogleGenerativeAI( - model= os.environ["LLM_PROVIDER"], + model=os.environ["LLM_PROVIDER"], temperature=0, # google_api_key="...", # or set env var: GOOGLE_API_KEY=... ) @@ -865,7 +845,7 @@ def init_session_state(defaults: dict) -> None: # st.stop() # Otherwise: render the provider picker + provider-specific fields - with st.expander("LLM Settings",expanded=st.session_state.creds_saved==False ): + with st.expander("LLM Settings", expanded=not st.session_state.creds_saved): provider = st.selectbox( "Choose LLM provider", options=["openai", "azure-openai", "gemini"], @@ -891,7 +871,7 @@ def init_session_state(defaults: dict) -> None: placeholder="sk-********************************" ) - if st.button("Save provider settings",type="primary", width="stretch"): + if st.button("Save provider settings", type="primary", width="stretch"): if not api_key or len(api_key) < 20: st.error("Please enter a valid OpenAI API key.") else: @@ -922,7 +902,7 @@ def init_session_state(defaults: dict) -> None: api_version = st.text_input("API version", value=pre_version, placeholder="2024-06-01") api_key = st.text_input("Azure OpenAI API key", type="password", value=pre_key, placeholder="****************") - if st.button("Save provider settings" ,type="primary", width="stretch"): + if st.button("Save provider settings", type="primary", width="stretch"): errs = [] if not endpoint.startswith("http"): errs.append("Endpoint must start with http/https.") @@ -949,9 +929,9 @@ def init_session_state(defaults: dict) -> None: # }) os.environ["LLM_PROVIDER"] = str(llm_provider.strip()) settings.LLM_PROVIDER = str(llm_provider.strip()) - os.environ["AZURE_OPENAI_API_KEY"]=str(api_key.strip()) - os.environ["OPENAI_API_VERSION"]=str(api_version.strip()) - os.environ["AZURE_OPENAI_ENDPOINT"]=str(endpoint.strip()) + os.environ["AZURE_OPENAI_API_KEY"] = str(api_key.strip()) + os.environ["OPENAI_API_VERSION"] = str(api_version.strip()) + os.environ["AZURE_OPENAI_ENDPOINT"] = str(endpoint.strip()) st.session_state.creds_saved = True st.success("Azure OpenAI settings saved.") @@ -968,7 +948,7 @@ def init_session_state(defaults: dict) -> None: ) api_key = st.text_input("Gemini API key", type="password", value=pre_key, placeholder="********************************") - if st.button("Save provider settings" , type="primary", width="stretch"): + if st.button("Save provider settings", type="primary", width="stretch"): if not api_key or len(api_key) < 20: st.error("Please enter a valid Gemini API key.") else: @@ -1013,7 +993,6 @@ def init_session_state(defaults: dict) -> None: st.rerun() - # ----------------------- ----------------------- ----------------------- ----------------------- ----------------------- # 5) Create a Semantic Model (uses all files in modified_input/) # ----------------------- ----------------------- ----------------------- ----------------------- ----------------------- @@ -1024,12 +1003,13 @@ def init_session_state(defaults: dict) -> None: def _list_modified_files(): return sorted(list(MODIFIED_DIR.glob("*.csv")) + list(MODIFIED_DIR.glob("*.xlsx"))) + def _source_type_from_suffix(p: Path) -> str: return "csv" if p.suffix.lower() == ".csv" else "excel" # Find modified files -modified_csv = list(MODIFIED_DIR.glob("*.csv")) +modified_csv = list(MODIFIED_DIR.glob("*.csv")) modified_xlsx = list(MODIFIED_DIR.glob("*.xlsx")) modified_files = modified_csv + modified_xlsx n_files = len(modified_files) @@ -1041,8 +1021,6 @@ def _source_type_from_suffix(p: Path) -> str: st.stop() - - if n_files == 0: st.info("No tables found. Upload & Freeze at least one table to continue.") elif st.session_state.get("route") != "semantic_all": @@ -1092,7 +1070,7 @@ def _source_type_from_suffix(p: Path) -> str: st.header("3) 🧠 Build Semantic Model") # 2) Define domain BEFORE using it st.markdown("#### Domain") - dcol1, dcol2 = st.columns([1,1]) + dcol1, dcol2 = st.columns([1, 1]) with dcol1: # show whatever is currently saved as the default in the box domain = st.text_input( @@ -1124,7 +1102,6 @@ def _source_type_from_suffix(p: Path) -> str: st.caption(f"Semantic Model will be built for **{len(files)}** table(s).") - gdf_iter_ph = st.empty() # optional: keep a cumulative copy if you still want it elsewhere st.session_state.setdefault("semantic_gdf_all", None) @@ -1133,10 +1110,10 @@ def _source_type_from_suffix(p: Path) -> str: my_bar = st.progress(0, text="Preparing data for semantic model creation. Please wait.") for idx, p in enumerate(files, start=1): # start=2 because first file already processed - if idx ==1: + if idx == 1: # Initialize with the first file first = files[0] - data_sources = { first.stem: {"path": str(first), "type": _source_type_from_suffix(first)} } + data_sources = {first.stem: {"path": str(first), "type": _source_type_from_suffix(first)}} with st.spinner(f"Initializing Semantic Model with `{first.name}` …"): sm = SemanticModel(data_input=data_sources, domain=domain.strip()) @@ -1161,13 +1138,12 @@ def _source_type_from_suffix(p: Path) -> str: # st.write(f"Generating glossary for `{ds_name}` …") # sm.generate_glossary() percent_complete = (idx / total) - my_bar.progress(percent_complete , text=f"Preparing data for semantic model creation {round(percent_complete*100)}%. Please wait.") + my_bar.progress(percent_complete, text=f"Preparing data for semantic model creation {round(percent_complete * 100)}%. Please wait.") try: profiling_df = sm.profiling_df.copy() glossary_df = sm.glossary_df.copy() - gdf_this = pd.merge( glossary_df, profiling_df, @@ -1182,21 +1158,19 @@ def _source_type_from_suffix(p: Path) -> str: st.toast(f"##### βœ… Glossary for `{ds_name}` ({idx}/{total})") # add title to the table st.markdown(f"### Glossary & Profiling details for `{idx}` tables") - st.dataframe(gdf_this[['table_name','column_name', 'datatype_l1', 'datatype_l2', 'count','null_count','distinct_count', - 'uniqueness', 'completeness', 'business_glossary', 'business_tags','sample_data']], + st.dataframe(gdf_this[['table_name', 'column_name', 'datatype_l1', 'datatype_l2', 'count', 'null_count', 'distinct_count', + 'uniqueness', 'completeness', 'business_glossary', 'business_tags', 'sample_data']], hide_index=True, width="stretch") st.session_state["sm"] = sm - except Exception as e: st.warning(f"Could not render glossary view for `{ds_name}`: {e}") st.success("Semantic model profiling & glossary generation completed for all tables.") - if st.session_state["data_profiling_glossary"] == False: + if not st.session_state["data_profiling_glossary"]: st.session_state["data_profiling_glossary"] = True st.rerun() - # Link prediction and back button (unchanged) st.divider() st.subheader("4) Link Prediction") @@ -1229,7 +1203,6 @@ def _source_type_from_suffix(p: Path) -> str: # links_list = predictor_instance.links # st.success("Link prediction completed.") - # try: # # ---------- usage ---------- # show_links_graph_and_table(links_list) # <-- pass your list[PredictedLink] here From f35598b98b67550923cebbd44d8a0c2c9e0b25d2 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Fri, 24 Oct 2025 17:32:47 +0530 Subject: [PATCH 05/26] Integrated retriever --- .../graph_based_column_search/retreiver.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py b/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py new file mode 100644 index 0000000..30d1052 --- /dev/null +++ b/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py @@ -0,0 +1,156 @@ +# import pickle +import _pickle as pickle +import heapq +import logging +import os +import sys + +import pandas as pd + +from qdrant_client import models + +from intugle.core import settings +from intugle.core.conceptual_search.graph_based_column_search.utils import ( + CONCEPTUAL_SEARCH_COLLECTION_NAME, +) +from intugle.core.conceptual_search.models import GraphFileName +from intugle.core.llms.embeddings import Embeddings, EmbeddingsType +from intugle.core.vector_store import AsyncQdrantService +from intugle.core.vector_store.qdrant import VectorSearchKwargs + +log = logging.getLogger(__name__) + + +class GraphSearch: + def __init__(self): + self.collection_name = CONCEPTUAL_SEARCH_COLLECTION_NAME + self.graph = self.fetch_graph_from_directory() + self.embedding_model = Embeddings( + model_name=settings.EMBEDDING_MODEL_NAME, + tokenizer_model=settings.TOKENIZER_MODEL_NAME, + ) + + @classmethod + def fetch_graph_from_directory(cls): + graph_path = os.path.join(settings.GRAPH_DIR, GraphFileName.FIELD) + + if os.path.exists(graph_path): + log.info(f"Graph file found at {graph_path}") + with open(graph_path, "rb") as _file: + graph = pickle.load(_file) + return graph + else: + log.error(f"Graph file not found at {graph_path}") + return None + + async def traverse_graph(self, query: str, graph, top_k=2, max_depth=2): + """ + Traverse the knowledge graph to find relevant information for the query. + """ + # Vectorize the query using the standard embedding model + vectors = await self.embedding_model.aencode( + [query], embeddings_types=[EmbeddingsType.LATE] + ) + query_vector = vectors[EmbeddingsType.LATE][0] + + vector_name = f"{self.embedding_model.model_name}-{EmbeddingsType.LATE}" + + async with AsyncQdrantService(collection_name=self.collection_name, collection_configurations={}) as qdb: + log.info(f"Traversing graph for query: {query}") + + search_result = await qdb.search( + query=query_vector, + search_using=vector_name, + includes=["metadata"], + search_kwargs=dict( + VectorSearchKwargs( + search_type="similarity", + top_k=sys.maxsize, + search_params=models.SearchParams( + hnsw_ef=sys.maxsize, + exact=True, + ), + ), + ), + ) + + if not search_result or not search_result.points: + return [], [], pd.DataFrame() + + points_data = [p.model_dump() for p in search_result.points] + search_df = pd.DataFrame(points_data) + + if search_df.empty: + return [], [], pd.DataFrame() + + search_df = pd.concat( + [search_df.drop("payload", axis=1), pd.json_normalize(search_df["payload"])], + axis=1, + ) + search_df = search_df[["score", "source", "row", "content"]] + search_df["score"] = search_df["score"] / len(query_vector) + search_df.set_index("row", inplace=True) + + similarities = list(search_df["score"].to_dict().items()) + similarities.sort(key=lambda x: x[1], reverse=True) + + starting_nodes = [node for node, _ in similarities[:top_k]] + log.info(f"Starting traversal from {len(starting_nodes)} nodes") + + visited = set() + traversal_path = [] + results = [] + queue = [] + + for node in starting_nodes: + # Find the similarity score for the node + score = next((s for n, s in similarities if n == node), 0) + heapq.heappush(queue, (-score, node)) + + while queue and len(results) < (top_k * 3): + _, node = heapq.heappop(queue) + if node in visited or node not in graph: + continue + + visited.add(node) + traversal_path.append(node) + results.append({"node_id": node, "source": graph.nodes[node]["source"]}) + + if len(traversal_path) < max_depth: + neighbors = [ + (neighbor, graph[node][neighbor]["weight"]) + for neighbor in graph.neighbors(node) + if neighbor not in visited + ] + for neighbor, weight in sorted(neighbors, key=lambda x: x[1], reverse=True): + heapq.heappush(queue, (-weight, neighbor)) + + log.info(f"Graph traversal found {len(results)} relevant chunks") + return results, traversal_path, search_df + + async def get_shortlisted_columns(self, query: str): + """ + Get shortlisted tables based on the query. + Args: + subscription_id (str): Subscription ID + query (str): User query + Returns: + List[Dict]: Shortlisted tables with relevant information + """ + + if self.graph is None: + log.error("Graph not found. Please create the graph first.") + return [] + + log.info("Graph loaded successfully") + + _, traversal_path, search_result = await self.traverse_graph( + query, + self.graph, + top_k=int(settings.NETWORKX_GRAPH_TOP_K_COLUMN), + max_depth=int(settings.NETWORKX_GRAPH_MAX_DEPTH_COLUMN), + ) + + shortlisted_columns = search_result.loc[[i for i in traversal_path]] + + return shortlisted_columns From e6e99917cf8fd082ffe5bd91f2a1d43163f1a46f Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Fri, 24 Oct 2025 17:33:10 +0530 Subject: [PATCH 06/26] settings networkx configs added --- src/intugle/core/settings.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index d511442..faed261 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -108,6 +108,12 @@ class Settings(BaseSettings): EMBEDDING_MODEL_NAME: str = "openai:ada" TOKENIZER_MODEL_NAME: str = "cl100k_base" + # NETWORKX GRAPH + NETWORKX_GRAPH_TOP_K_COLUMN: int = 4 + NETWORKX_GRAPH_MAX_DEPTH_COLUMN: int = 4 + NETWORKX_GRAPH_TOP_K_TABLE: int = 2 + NETWORKX_GRAPH_MAX_DEPTH_TABLE: int = 2 + model_config = SettingsConfigDict( env_file=f"{BASE_PATH}/.env", env_file_encoding="utf-8", From 9cb3b493ccae91c84d31bfb6c801c3e9f635a5d7 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Sat, 25 Oct 2025 14:36:15 +0530 Subject: [PATCH 07/26] Added table graph kg initialization --- .../graph_based_column_search/utils.py | 45 ++-- .../graph_based_table_search/__init__.py | 0 .../networkx_initializers.py | 83 ++++++ .../graph_based_table_search/retreiver.py | 150 +++++++++++ .../graph_based_table_search/utils.py | 243 ++++++++++++++++++ src/intugle/core/conceptual_search/utils.py | 52 ++-- src/intugle/core/settings.py | 4 +- 7 files changed, 526 insertions(+), 51 deletions(-) create mode 100644 src/intugle/core/conceptual_search/graph_based_table_search/__init__.py create mode 100644 src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py create mode 100644 src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py create mode 100644 src/intugle/core/conceptual_search/graph_based_table_search/utils.py diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py index eb34360..90c384d 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py @@ -16,8 +16,8 @@ from intugle.core.conceptual_search.models import QdrantCollectionSuffix from intugle.core.conceptual_search.utils import ( batched, - extract_concepts_column, fetch_column_with_description, + manual_concept_extraction, ) from intugle.core.llms.chat import ChatModelLLM from intugle.core.llms.embeddings import Embeddings, EmbeddingsType @@ -162,9 +162,33 @@ def remove_unwanted_concepts(concepts: list, filter_out: list): return list(filter(lambda concept: concept not in filter_out, concepts)) +def extract_concepts_column(text, llm): + """ + Extract key concepts from a column description. + """ + system_message = """ + # Instructions: + - Extract key concepts and entities from the provided description of a field in a database. + - Return ONLY a list of 2 key terms, entities, or concepts that are most important in this text. + - Return a valid JSON array of strings. Example:["Entity1", "Entity2"] + - Do not mention table names or the field name itself, or irrelevant terms. + - The list of terms should be unique. + """ + + messages = [ + ("system", system_message), + ("user", f"Extract key concepts from field description:\n\n{text[:3000]}"), + ] + try: + ai_msg = llm.invoke(messages) + return manual_concept_extraction(ai_msg) + except Exception as e: + log.error(f"Concept extraction for column failed: {e}") + return [] + def prepare_chunk_document(manifest): - column_description_path = os.path.join(settings.COL_DESCRIPTION_DIR, "column_description.csv") + column_description_path = os.path.join(settings.DESCRIPTIONS_DIR, "column_description.csv") column_description_path = Path(column_description_path) column_description_path.parent.mkdir(exist_ok=True, parents=True) @@ -174,18 +198,12 @@ def prepare_chunk_document(manifest): ) if not column_description_path.exists(): - # existing = pd.read_csv(column_description_path) - column_description_df = fetch_column_with_description(manifest) column_description_df['Text'] = column_description_df['column_name'] + " - " + column_description_df['business_glossary'] column_description_df['id'] = column_description_df['table_name'] + "$$##$$" + column_description_df['column_name'] - # remaining_columns = set(column_description_df["id"].unique()) - set(existing["id"].unique()) - - # column_description_df = column_description_df[(column_description_df["business_glossary"] != "") & (column_description_df["id"].isin(list(remaining_columns)))].reset_index(drop=True) - column_description_df = column_description_df[(column_description_df["business_glossary"] != "")].reset_index(drop=True) log.info(f"[*] Extracting concepts for {column_description_df.shape[0]} columns") @@ -194,20 +212,14 @@ def prepare_chunk_document(manifest): column_description_df.loc[data.index, "concepts"] = data["Text"].progress_apply(lambda text: clean_concepts(extract_concepts_column(text=text, llm=llm))) - # column_description_df.loc[data.index,"concepts"] = data["business_tags"].apply(clean_concepts) - column_description_df.loc[data.index, "concepts"] = column_description_df.loc[data.index].apply(lambda row: remove_unwanted_concepts(concepts=row["concepts"], filter_out=[row['column_name'], 'sap erp', 'sap erp system', 'sap']), axis=1) - - # breakpoint() column_description_df.loc[data.index].to_csv(column_description_path, index=False, mode='a', header=not os.path.exists(column_description_path)) - log.info(f"[*] Batch {i} completed ... sleeping for 60 seconds") - - # time.sleep(60) + log.info(f"[*] Batch {i} completed") else: @@ -217,7 +229,6 @@ def prepare_chunk_document(manifest): concepts_to_be_removed = filter_out_concepts(data=column_description_df[["concepts"]]) - # ["unique identifier","transaction","transaction code"] column_description_df["concepts"] = column_description_df.apply(lambda row: remove_unwanted_concepts(concepts=row["concepts"], filter_out=concepts_to_be_removed), axis=1) data_documents = [] @@ -226,4 +237,4 @@ def prepare_chunk_document(manifest): data_documents.append(VDocument(page_content=row['Text'].strip().lower(), metadata={"source": row['id'], "row": index, "concepts": row['concepts']})) - return data_documents \ No newline at end of file + return data_documents diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/__init__.py b/src/intugle/core/conceptual_search/graph_based_table_search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py new file mode 100644 index 0000000..59d921f --- /dev/null +++ b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py @@ -0,0 +1,83 @@ +import asyncio +import logging +import os +import pickle + +import networkx as nx + +from intugle.core import settings +from intugle.core.conceptual_search.graph_based_table_search.utils import ( + create_embeddings, + prepare_chunk_document, +) +from intugle.core.conceptual_search.models import GraphFileName +from intugle.core.conceptual_search.utils import colbert_score_numpy + +log = logging.getLogger(__name__) + + +def build_knowledge_graph(doc): + """ + Build a knowledge graph from text chunks. + """ + log.info("Building knowledge graph for tables...") + + graph = nx.Graph() + + log.info("Creating embeddings for table chunks...") + embeddings = asyncio.run(create_embeddings(doc)) + + log.info("Adding nodes to the graph...") + for i, chunk in enumerate(doc): + graph.add_node(i, source=chunk.metadata['source']) + log.info(f"Node {i} added for source: {chunk.metadata['source']}") + + log.info("Creating edges between nodes...") + for i in range(len(doc)): + concepts_i = doc[i].metadata["concepts"] + node_concepts = set(concepts_i) + + for j in range(i + 1, len(doc)): + other_concepts = set(doc[j].metadata['concepts']) + shared_concepts = node_concepts.intersection(other_concepts) + + if shared_concepts: + similarity = colbert_score_numpy(embeddings[i], embeddings[j]) + concept_score = len(shared_concepts) / min(len(node_concepts), len(other_concepts)) + edge_weight = 0.7 * similarity + 0.3 * concept_score + + if edge_weight > 0.7: + graph.add_edge( + i, j, + weight=edge_weight, + similarity=similarity, + shared_concepts=list(shared_concepts) + ) + + log.info(f"Knowledge graph built with {graph.number_of_nodes()} nodes and {graph.number_of_edges()} edges") + return graph, embeddings + + +def prepare_networkx_graph(manifest): + + graph_path = os.path.join(settings.GRAPH_DIR, GraphFileName.TABLE) + + if os.path.exists(graph_path): + print('[!] Table search graph already built... skipping the step') + return + + docs = prepare_chunk_document(manifest) + + graph, _ = build_knowledge_graph(doc=docs) + + os.makedirs(settings.GRAPH_DIR, exist_ok=True) + + try: + with open(graph_path, "wb") as _file: + pickle.dump(graph, _file, pickle.HIGHEST_PROTOCOL) + except Exception as ex: + log.error(f"Failed to save graph: {ex}") + raise ex + + print(f"Graph saved to {graph_path}") + print(f"Graph created with {graph.number_of_nodes()} nodes and {graph.number_of_edges()} edges") diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py b/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py new file mode 100644 index 0000000..dec86b0 --- /dev/null +++ b/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py @@ -0,0 +1,150 @@ +import _pickle as pickle +import heapq +import logging +import os +import sys + +import pandas as pd + +from qdrant_client import models + +from intugle.core import settings +from intugle.core.conceptual_search.graph_based_table_search.utils import ( + CONCEPTUAL_SEARCH_COLLECTION_NAME, +) +from intugle.core.conceptual_search.models import GraphFileName +from intugle.core.llms.embeddings import Embeddings, EmbeddingsType +from intugle.core.vector_store import AsyncQdrantService +from intugle.core.vector_store.qdrant import VectorSearchKwargs + +log = logging.getLogger(__name__) + + +class GraphSearch: + def __init__(self): + self.collection_name = CONCEPTUAL_SEARCH_COLLECTION_NAME + self.graph = self.fetch_graph_from_directory() + self.embedding_model = Embeddings( + model_name=settings.EMBEDDING_MODEL_NAME, + tokenizer_model=settings.TOKENIZER_MODEL_NAME, + ) + + @classmethod + def fetch_graph_from_directory(cls): + graph_path = os.path.join(settings.GRAPH_DIR, GraphFileName.TABLE) + + if os.path.exists(graph_path): + log.info(f"Graph file found at {graph_path}") + with open(graph_path, "rb") as _file: + graph = pickle.load(_file) + return graph + else: + log.error(f"Graph file not found at {graph_path}") + return None + + async def traverse_graph(self, query: str, graph, top_k=2, max_depth=2): + """ + Traverse the knowledge graph to find relevant information for the query. + """ + vectors = await self.embedding_model.aencode( + [query], embeddings_types=[EmbeddingsType.LATE] + ) + query_vector = vectors[EmbeddingsType.LATE][0] + + vector_name = f"{self.embedding_model.model_name}-{EmbeddingsType.LATE}" + + async with AsyncQdrantService(collection_name=self.collection_name, collection_configurations={}) as qdb: + log.info(f"Traversing graph for query: {query}") + + search_result = await qdb.search( + query=query_vector, + search_using=vector_name, + includes=["metadata"], + search_kwargs=dict( + VectorSearchKwargs( + search_type="similarity", + top_k=sys.maxsize, + search_params=models.SearchParams( + hnsw_ef=sys.maxsize, + exact=True, + ), + ), + ), + ) + + if not search_result or not search_result.points: + return [], [], pd.DataFrame() + + points_data = [p.model_dump() for p in search_result.points] + search_df = pd.DataFrame(points_data) + + if search_df.empty: + return [], [], pd.DataFrame() + + search_df = pd.concat( + [search_df.drop("payload", axis=1), pd.json_normalize(search_df["payload"])], + axis=1, + ) + search_df = search_df[["score", "source", "row", "content"]] + search_df["score"] = search_df["score"] / len(query_vector) + search_df.set_index("row", inplace=True) + + similarities = list(search_df["score"].to_dict().items()) + similarities.sort(key=lambda x: x[1], reverse=True) + + starting_nodes = [node for node, _ in similarities[:top_k]] + log.info(f"Starting traversal from {len(starting_nodes)} nodes") + + visited = set() + traversal_path = [] + results = [] + queue = [] + + for node in starting_nodes: + score = next((s for n, s in similarities if n == node), 0) + heapq.heappush(queue, (-score, node)) + + while queue and len(results) < (top_k * 3): + _, node = heapq.heappop(queue) + if node in visited or node not in graph: + continue + + visited.add(node) + traversal_path.append(node) + results.append({"node_id": node, "source": graph.nodes[node]["source"]}) + + if len(traversal_path) < max_depth: + neighbors = [ + (neighbor, graph[node][neighbor]["weight"]) + for neighbor in graph.neighbors(node) + if neighbor not in visited + ] + for neighbor, weight in sorted(neighbors, key=lambda x: x[1], reverse=True): + heapq.heappush(queue, (-weight, neighbor)) + + log.info(f"Graph traversal found {len(results)} relevant chunks") + return results, traversal_path, search_df + + async def get_shortlisted_tables(self, query: str): + """ + Get shortlisted tables based on the query. + """ + if self.graph is None: + log.error("Graph not found. Please create the graph first.") + return pd.DataFrame() + + log.info("Graph loaded successfully") + + _, traversal_path, search_result = await self.traverse_graph( + query, + self.graph, + top_k=int(settings.NETWORKX_GRAPH_TOP_K_TABLE), + max_depth=int(settings.NETWORKX_GRAPH_MAX_DEPTH_TABLE), + ) + + if not traversal_path: + return pd.DataFrame() + + shortlisted_tables = search_result.loc[search_result.index.isin(traversal_path)] + + return shortlisted_tables diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py new file mode 100644 index 0000000..144f946 --- /dev/null +++ b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py @@ -0,0 +1,243 @@ +import ast +import logging +import os + +from itertools import chain +from pathlib import Path +from uuid import uuid4 + +import numpy as np +import pandas as pd + +from qdrant_client import models +from tqdm import tqdm + +from intugle.core import settings +from intugle.core.conceptual_search.models import QdrantCollectionSuffix +from intugle.core.conceptual_search.utils import ( + batched, + fetch_table_with_description, + manual_concept_extraction, +) +from intugle.core.llms.chat import ChatModelLLM +from intugle.core.llms.embeddings import Embeddings, EmbeddingsType +from intugle.core.vector_store import AsyncQdrantService, VDocument +from intugle.core.vector_store.qdrant import QdrantVectorConfiguration + +log = logging.getLogger(__name__) + +tqdm.pandas() + +CONCEPTUAL_SEARCH_COLLECTION_NAME = "conceptual_search_tables" + + +def collection_name(prefix: str, suffix: str = QdrantCollectionSuffix.TABLE): + + return f"{prefix}{suffix}" + + +async def fetch_embeddings_from_qdrant(collection_name): + + data = pd.DataFrame() + + async with AsyncQdrantService(collection_name=collection_name, collection_configurations={}) as qdb: + + data = await qdb.get(includes=["metadata", "embeddings"], return_document=False) + + data = pd.DataFrame([d.model_dump() for d in data])[["payload", "vector"]] + + if data.shape[0] > 0: + + data = pd.concat( + [data, pd.json_normalize(data.payload)], axis=1 + )[["row", "source", "vector", "content"]] + + if "row" in data.columns: + + return data.sort_values(by="row", ascending=True).reset_index(drop=True) + + return data + + +async def create_embeddings(doc: list[VDocument], recreate: bool = False): + """ + Create and store embeddings for the given documents using the configured embedding model. + This function aligns with the intugle architecture for handling embeddings and vector stores. + """ + embedding_model = Embeddings( + model_name=settings.EMBEDDING_MODEL_NAME, + tokenizer_model=settings.TOKENIZER_MODEL_NAME, + ) + + vectors_config = { + f"{embedding_model.model_name}-{EmbeddingsType.LATE}": models.VectorParams( + size=embedding_model.embeddings_size, + distance=models.Distance.COSINE, + multivector_config=models.MultiVectorConfig( + comparator=models.MultiVectorComparator.MAX_SIM + ), + ), + } + configuration = QdrantVectorConfiguration(vectors_config=vectors_config) + + async with AsyncQdrantService( + collection_name=CONCEPTUAL_SEARCH_COLLECTION_NAME, + collection_configurations=configuration, + ) as qdb: + if recreate: + await qdb.delete_collection() + await qdb.create_collection() + log.info(f"[*] Deleted and recreated collection {CONCEPTUAL_SEARCH_COLLECTION_NAME}") + + count_result = await qdb.count() + count_req = count_result.count + if count_req < len(doc): + log.info("Vectorizing and adding documents to Qdrant...") + vectors = await embedding_model.aencode( + [d.page_content for d in doc], + embeddings_types=[EmbeddingsType.LATE], + ) + late_vectors = vectors[EmbeddingsType.LATE] + + points = [] + for i, document in enumerate(doc): + point_id = str(uuid4()) + vector = {f"{embedding_model.model_name}-{EmbeddingsType.LATE}": late_vectors[i]} + payload = {**document.metadata, "content": document.page_content} + points.append( + models.PointStruct( + id=point_id, + vector=vector, + payload=payload, + ) + ) + + for batch in batched(points, 30): + qdb.bulk_insert(points=batch) + log.info("Embeddings created and inserted into Qdrant DB") + + log.info("Fetching embeddings from Qdrant...") + all_vector_records = await fetch_embeddings_from_qdrant( + collection_name=CONCEPTUAL_SEARCH_COLLECTION_NAME + ) + log.info("Embeddings fetched from Qdrant.") + + vector_name = f"{embedding_model.model_name}-{EmbeddingsType.LATE}" + all_embeddings = [ + np.array(item["vector"][vector_name]) + for _, item in all_vector_records.iterrows() + ] + + return all_embeddings + + +def filter_out_concepts(data: pd.DataFrame, threshold: float = 0.1): + total_documents = data.shape[0] + concepts = pd.DataFrame(list(chain(*data.concepts.tolist())), columns=["concepts"]) + value_counts = concepts.value_counts().reset_index() + value_counts["proportion"] = value_counts["count"] / total_documents + + return value_counts.loc[value_counts.proportion > threshold].concepts.tolist() + + +def clean_concepts(concepts: list): + + if isinstance(concepts, list): + + return list(map(lambda concept: concept.strip().lower(), concepts)) + + return concepts + + +def remove_unwanted_concepts(concepts: list, filter_out: list): + + return list(filter(lambda concept: concept not in filter_out, concepts)) + + +def extract_concepts_table(text, llm): + """ + Extract key concepts from a table description. + """ + system_message = """ + # Instructions: + - Extract key concepts and entities from the provided description of a database table. + - Return ONLY a list of 1-5 key terms, entities, or concepts that are most important in this text. + - Return a valid JSON array of strings. Example:["Entity1", "Entity2"] + - Do not mention table names or irrelevant terms. + - The list of terms should be unique. + """ + messages = [ + ("system", system_message), + ("user", f"Extract key concepts from table description:\n\n{text[:3000]}"), + ] + try: + ai_msg = llm.invoke(messages) + return manual_concept_extraction(ai_msg) + except Exception as e: + log.error(f"Concept extraction for table failed: {e}") + return [] + + +def prepare_chunk_document(manifest): + table_description_path = os.path.join(settings.DESCRIPTIONS_DIR, "table_description.csv") + table_description_path = Path(table_description_path) + table_description_path.parent.mkdir(exist_ok=True, parents=True) + + llm = ChatModelLLM.get_llm( + model_name=settings.LLM_PROVIDER, + llm_config={"temperature": 0.05}, + ) + if not table_description_path.exists(): + + table_description_df = fetch_table_with_description(manifest) + + def build_table_description(row:pd.Series): + + description = [] + + domain = row["domain_name"] + + if not pd.isna(domain) and len(domain.strip()) != 0: + + description.append(f"Table Domain: {domain.strip()}") + + description += [f"Table Name: {row['table_name']}",f"Table Description: {row['table_description']}"] + + return "\n".join(description) + + table_description_df['table_description'] = table_description_df.apply(build_table_description,axis=1) + table_description_df['id'] = table_description_df['table_name'] + + log.info("[*] Extracting concepts for tables") + + for i,data in tqdm(enumerate(list(batched(table_description_df,30)),1)): + + concepts = data["table_description"].apply(extract_concepts_table,llm=llm) + + table_description_df.loc[data.index,"concepts"] = concepts + + log.info(f"[*] Batch {i} completed") + + table_description_df["concepts"] = table_description_df.concepts.apply(clean_concepts) + + table_description_df.to_csv(table_description_path,index=False) + + else: + + table_description_df = pd.read_csv(table_description_path) + + table_description_df["concepts"] = table_description_df.concepts.apply(ast.literal_eval) + + def remove_concepts(concepts:list,to_remove=["hvac industry"]): + + return list(filter(lambda concept:concept.strip().lower() not in to_remove,concepts)) + + table_description_df["concepts"] = table_description_df.concepts.apply(remove_concepts) + + data_documents = [] + + for index, row in table_description_df.iterrows(): + + data_documents.append(VDocument(page_content=row['table_description'].strip().lower(), metadata={"source": row['id'], "row": index,"concepts":row['concepts']})) + + return data_documents \ No newline at end of file diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index bcd2e1a..b367ae3 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -51,7 +51,6 @@ def colbert_score_numpy( def manual_concept_extraction(ai_msg): try: - # Try to parse a clean JSON array of strings concepts = json.loads(ai_msg.content) if isinstance(concepts, list) and all(isinstance(c, str) for c in concepts): return concepts @@ -63,7 +62,6 @@ def manual_concept_extraction(ai_msg): # --- Fallback strategy --- try: raw_text = getattr(ai_msg, "content", "") - # Attempt to extract a list manually from the raw text matches = re.findall(r"\[(.*?)\]", raw_text, re.DOTALL) if matches: items = re.findall(r'"([^"]+)"', matches[0]) @@ -71,48 +69,38 @@ def manual_concept_extraction(ai_msg): else: log.info("Fallback regex parsing did not find any matches.") except Exception as ex: - log.error(f"Fallback regex parsing didnot find any matches\nReason: {ex}") + log.error(f"Fallback regex parsing failed: {ex}") return [] -def extract_concepts_column(text, llm): +def fetch_table_with_description(manifest: "Manifest") -> pd.DataFrame: """ - Extract key concepts from text using GROQ's API. - - Args: - text (str): Text to extract concepts from - - Returns: - List[str]: List of key concepts or entities - """ - system_message = """ - # Instructions: - - Extract key concepts and entities from the provided description of a field in database . - - Return ONLY a list of 2 key terms, entities, or concepts that are most important in this text. - - Return a valid JSON array of strings. Example:["Entity1", "Entity2"] - - Donot mention table names or field name itself, or irrelevant terms , irrelevant entities or irreleavant concepts. - - The list of terms should be unique. + Fetches all table details from the manifest. """ + table_data = [] + for source in manifest.sources.values(): + table = source.table + if not table: + continue - messages = [ - ("system", system_message), - ("user", f"Extract key concepts from field description:\n\n{text[:3000]}"), - ] - try: - ai_msg = llm.invoke(messages) - concepts = manual_concept_extraction(ai_msg) - return concepts - except Exception as fallback_err: - log.info(f"Fallback parsing also failed: {fallback_err}") + table_data.append( + { + "table_name": table.name, + "table_description": table.description or "", + "domain_name": source.schema, # Using schema as domain for now + } + ) - return [] + if not table_data: + return pd.DataFrame(columns=["table_name", "table_description", "domain_name"]) + + return pd.DataFrame(table_data) def fetch_column_with_description(manifest: "Manifest") -> pd.DataFrame: """ Fetches all column details from the manifest. - This replaces the original SQL-based implementation. """ column_data = [] for source in manifest.sources.values(): @@ -144,4 +132,4 @@ def fetch_column_with_description(manifest: "Manifest") -> pd.DataFrame: ] ) - return pd.DataFrame(column_data) \ No newline at end of file + return pd.DataFrame(column_data) diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index faed261..43ab181 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -45,8 +45,8 @@ class Settings(BaseSettings): GRAPH_DIR_NAME: str = "kg" MODELS_DIR: str = os.path.join(PROJECT_BASE, MODELS_DIR_NAME) GRAPH_DIR: str = os.path.join(PROJECT_BASE, GRAPH_DIR_NAME) - COL_DESCRIPTION_DIR: str = os.path.join(PROJECT_BASE, "column_descriptions") - + DESCRIPTIONS_DIR: str = os.path.join(PROJECT_BASE, "descriptions") + PROFILES_PATH: str = os.path.join(os.getcwd(), "profiles.yml") PROFILES: dict = load_profiles_configuration(PROFILES_PATH) From 3d0f3e15ed1ff64e205c46f599f9453740981f9c Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Sat, 25 Oct 2025 15:42:39 +0530 Subject: [PATCH 08/26] Agent integration testing --- pyproject.toml | 1 + .../conceptual_search/agent/initializer.py | 24 ++ .../core/conceptual_search/agent/prompts.py | 81 ++++ .../conceptual_search/agent/retrievers.py | 78 ++++ .../agent/tools/tool_builder.py | 386 ++++++++++++++++++ .../agent/tools/web_tools.py | 35 ++ .../graph_based_column_search/utils.py | 1 + .../graph_based_table_search/utils.py | 20 +- src/intugle/core/conceptual_search/search.py | 114 ++++++ src/intugle/core/conceptual_search/utils.py | 2 +- src/intugle/core/settings.py | 1 + 11 files changed, 732 insertions(+), 11 deletions(-) create mode 100644 src/intugle/core/conceptual_search/agent/initializer.py create mode 100644 src/intugle/core/conceptual_search/agent/prompts.py create mode 100644 src/intugle/core/conceptual_search/agent/retrievers.py create mode 100644 src/intugle/core/conceptual_search/agent/tools/tool_builder.py create mode 100644 src/intugle/core/conceptual_search/agent/tools/web_tools.py create mode 100644 src/intugle/core/conceptual_search/search.py diff --git a/pyproject.toml b/pyproject.toml index 0d3182b..5ecfc63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "rich>=14.1.0", "aiohttp>=3.9.5", "aiofiles>=23.2.1", + "tavily-python>=0.1.11", ] [project.optional-dependencies] diff --git a/src/intugle/core/conceptual_search/agent/initializer.py b/src/intugle/core/conceptual_search/agent/initializer.py new file mode 100644 index 0000000..84fa4a1 --- /dev/null +++ b/src/intugle/core/conceptual_search/agent/initializer.py @@ -0,0 +1,24 @@ +from typing import List + +from langchain_core.runnables import RunnableSerializable +from langchain_core.tools import StructuredTool +from langgraph.prebuilt import create_react_agent + +from intugle.core.conceptual_search.agent.prompts import ( + data_product_builder_prompt, + data_product_planner_prompt, +) +from intugle.core.conceptual_search.agent.tools.web_tools import web_search + + +def data_product_planner_agent(llm, tools: List[StructuredTool]) -> RunnableSerializable: + agent = create_react_agent( + llm, tools + [web_search], prompt=data_product_planner_prompt + ) + + return agent + + +def data_product_builder_agent(llm, tools: List[StructuredTool]) -> RunnableSerializable: + agent = create_react_agent(llm, tools=tools, prompt=data_product_builder_prompt) + return agent diff --git a/src/intugle/core/conceptual_search/agent/prompts.py b/src/intugle/core/conceptual_search/agent/prompts.py new file mode 100644 index 0000000..dde6fe9 --- /dev/null +++ b/src/intugle/core/conceptual_search/agent/prompts.py @@ -0,0 +1,81 @@ +from langchain_core.prompts import ChatPromptTemplate + +data_product_planner_prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + "You are a data product assistant. Your task is to build a high-quality list of Dimensions and Measures for a business data product.\n\n" + "Follow these steps:\n" + "1. Use `retrieve_existing_data_products(statement)` to identify similar data products and gather candidate Dimensions and Measures.\n" + "2. Use `retrieve_table_details(statement)` to get relevant database tables. If the initial query does not return all required information, do not hesitate to run multiple queries with refined or different search terms (e.g., 'customer data', 'order transactions', 'technician schedule').\n" + "3. Use `web_search(question)` only to get broader ideas or industry best practices β€” do not use it for final attribute definitions.\n\n" + "When creating the list of attributes:\n" + "- Ensure each Dimension or Measure is grounded in the retrieved tables (i.e., it must be derivable from available data).\n" + "- Do NOT invent attributes that cannot be linked to existing fields or standard business metrics.\n" + "- Avoid generalities β€” prefer specific, field-aligned attributes over vague concepts.\n" + "- Use precise, unambiguous names.\n" + "- Eliminate duplicates and redundancy (e.g., avoid 'Total Revenue' and 'Revenue Total').\n" + "- Group Measures and Dimensions logically if possible.\n" + "- Include a short description for each.\n" + "- Tag each as either a 'Dimension' or a 'Measure'.\n\n" + "Once ready, persist the final list using `save_data_product(attribute_data)` in the following format:\n", + ), + ("human", "{messages}"), + ] +) + +tagging_prompt = ChatPromptTemplate.from_template( + """ +You are a data relevance evaluator. Your task is to score how relevant a given database column is to a specific attribute required for building a data product. + +Use the information below and output a relevance score between 1 and 10, where: +- 10 = highly relevant (perfect match), +- 1 = not relevant at all. + +Please consider name similarity, semantic meaning, and context from the table description and DDL. + +------------------------------------------------------------ + +**Data Product Name**: {data_product_name} + +**Required Attribute**: {attribute_name} + +**Attribute Description**: {attribute_description} + +**Attribute Type**: {attribute_type} # Dimension or Measure + +**Candidate Column**: {column_name} + +**Candiate Column Description**: {column_description} + +**Table Description**: {table_description} + +------------------------------------------------------------ + +Evaluate how relevant the candidate column is to the required attribute. + +Return ONLY the following in your response: +Relevance Score (1-10): +""" +) + + +data_product_builder_prompt = """You are a data exploration assistant. You are tasked with identifying the correct column(s) across available data sources that satisfy the attribute definition: + +You have access to the following tools: +list_of_tables: Retrieve a list of unique tables from the dataset along with their glossary and domain information. +column_logic_store: Stores logic used to derive an attribute from one or more column-table combinations. +column_retriever: Retrieves relevant columns from the provided list of tables that semantically match the given attribute name and description. + +Use the following strategy: +- Iteratively explore all available tables using the list_of_tables tool. +- For relevant tables, use column_retriever to examine column names and column descriptions. +- Continue the process until a column (or combination of columns) clearly maps to the given attribute description. +- If no single column is sufficient, determine a transformation logic using a combination of related columns. +- Do not stop until either: + -- The attribute is matched with high confidence. + -- All possibilities are exhausted and it's determined that the attribute cannot be constructed with available data. +- At the end, store the matched column(s) and their corresponding tables. The data transformation logic needed to compute the final measure or dimension (in SQL or pseudo-code). + +**Attribute Details*** +""" diff --git a/src/intugle/core/conceptual_search/agent/retrievers.py b/src/intugle/core/conceptual_search/agent/retrievers.py new file mode 100644 index 0000000..6e93f0a --- /dev/null +++ b/src/intugle/core/conceptual_search/agent/retrievers.py @@ -0,0 +1,78 @@ +import asyncio +import logging + +import pandas as pd + +from langchain_core.documents import Document + +from intugle.core.conceptual_search.graph_based_column_search.retreiver import ( + GraphSearch as ColumnGraphSearch, +) +from intugle.core.conceptual_search.graph_based_table_search.retreiver import ( + GraphSearch as TableGraphSearch, +) + +log = logging.getLogger(__name__) + + +class ConceptualSearchRetrievers: + def to_documents(self, results: pd.DataFrame) -> list[Document]: + docs = [] + if results.empty: + return docs + + for _, row in results.iterrows(): + source = row.get("source", "") + content = row.get("content", "") + table_column = source.split("$$##$$") + + if len(table_column) > 1: + table, column = table_column + meta_data = {"table": table, "column": column} + else: + meta_data = {"table": table_column[0]} + + docs.append(Document(page_content=content, metadata=meta_data)) + + return docs + + def __init__(self): + self.table_graph = TableGraphSearch() + self.column_graph = ColumnGraphSearch() + + def data_products_retriever( + self, + query: str, + ) -> list[Document]: + """ + Retrieves existing data products. + NOTE: This is a placeholder and currently returns no documents. + """ + log.warning( + f"Attempted to retrieve data products for query: '{query}'. This feature is not yet implemented." + ) + return [] + + def table_retriever(self, query: str) -> list[Document]: + results = asyncio.run(self.table_graph.get_shortlisted_tables(query)) + return self.to_documents(results) + + def column_retriever( + self, attribute_name: str, attribute_description: str = "" + ) -> list[Document]: + results_attribute_name = asyncio.run( + self.column_graph.get_shortlisted_columns(query=attribute_name) + ) + + results_attribue_description = asyncio.run( + self.column_graph.get_shortlisted_columns(query=attribute_description) + ) + + if results_attribute_name.empty and results_attribue_description.empty: + return [] + + results = pd.concat( + [results_attribute_name, results_attribue_description] + ).drop_duplicates() + + return self.to_documents(results=results) diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py new file mode 100644 index 0000000..88fb359 --- /dev/null +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -0,0 +1,386 @@ +import csv +import logging +import os + +from typing import Annotated, Any, Dict, List, Optional + +import pandas as pd + +from langchain_core.documents import Document +from langchain_core.tools import StructuredTool + +from intugle.core.conceptual_search.agent.retrievers import ( + ConceptualSearchRetrievers, +) +from intugle.core.conceptual_search.utils import ( + extract_data_product_info, + extract_table_details, + fetch_table_with_description, +) +from intugle.parser.manifest import Manifest + +log = logging.getLogger(__name__) + + +class DataProductPlannerAgentTools: + def __init__(self, retrieval_tool: ConceptualSearchRetrievers): + self.retrieval_tool = retrieval_tool + + def list_tools( + self, + ) -> List[StructuredTool]: + return [ + StructuredTool.from_function( + name="retrieve_existing_data_products", + func=self.retrieve_existing_data_products, + description="""Retrieve dimensions and measures for similar existing data products. + + This function uses a retriever (retriever_dp) to fetch documents describing + similar data products based on the input statement. It then extracts the + data product name, its dimensions, and its measures. Includes error handling. + + Returns: + List[Dict[str, Any]]: A list of dictionaries. Each dictionary represents a found + data product and contains the keys 'Data_Product' (str), + 'Dimensions' (List[str]), and 'Measures' (List[str]). + Returns an empty list ([]) if no similar products are found + or if a critical error occurs during the process. + """, + ), + StructuredTool.from_function( + name="retrieve_table_details", + func=self.retrieve_table_details, + description=""" + Retrieve table details from a database based on a natural language statement. + + This function uses a retriever to fetch relevant table information based on the + provided statement, then extracts structured details. It includes error handling + for the retrieval and extraction steps. + + Returns: + List[Dict[str, Any]]: A list of dictionaries, where each dictionary contains + extracted details of a potentially relevant table (e.g., + {'Table_Name': ..., 'Description_Snippet': ...}). + Returns an empty list ([]) if no relevant tables are + found or if a critical error occurs during retrieval/extraction. + """, + ), + StructuredTool.from_function( + self.save_data_product, + name="save_data_product", + description=""" + Processes a list of attribute dictionaries and saves them to a CSV file, + with detailed print statements for monitoring. + + Each dictionary in the input list should represent a single attribute, + with the attribute name as the key and a list containing + [description, classification ('Dimension'/'Measure')] as the value. + + Returns: + A string message indicating success or failure. + """, + ), + ] + + def retrieve_existing_data_products( + self, + statement: Annotated[ + str, + """Name or description of the data product you want to search for + (e.g., 'sales dashboard', 'customer churn analysis').""", + ], + ) -> List[Dict[str, Any]]: + log.info(f"--- Executing retrieve_existing_data_products for statement: '{statement}' ---") + try: + documents = self.retrieval_tool.data_products_retriever(statement) + log.info(f"Retriever returned {len(documents)} potential document(s).") + + if not documents: + log.info("No similar data product documents were found by the retriever.") + return ["No similar data product documents were found by the retriever."] + + result = extract_data_product_info(documents) + log.info(f"Extraction process resulted in {len(result)} structured data product detail(s).") + return result + except Exception as e: + log.error(f"ERROR: An exception occurred during data product retrieval: {e}", exc_info=True) + return [f"ERROR: An exception occurred during data product retrieval: {e}"] + + def retrieve_table_details( + self, + statement: Annotated[ + str, + """A natural language description of the table name and/or + purpose you are searching for (e.g., "customer information table", + "table containing order line items").""", + ], + ) -> List[Dict[str, Any]]: + log.info(f"--- Executing retrieve_table_details for statement: '{statement}' ---") + try: + documents = self.retrieval_tool.table_retriever(statement) + log.info(f"Retriever returned {len(documents)} potential document(s).") + + if not documents: + log.info("No relevant table documents were found by the retriever for the given statement.") + return ["No relevant table documents were found by the retriever for the given statement."] + + result = extract_table_details(documents) + log.info(f"Extraction process resulted in {len(result)} structured table detail(s).") + return result + except Exception as e: + log.error(f"ERROR: An exception occurred during detail extraction: {e}", exc_info=True) + return [f"ERROR: An exception occurred during detail extraction: {e}"] + + def save_data_product( + self, + data_product_name: Annotated[str, "Name of the data product"], + data_product_description: Annotated[ + str, "Short description of the data product" + ], + attribute_data: Annotated[ + List[Dict[str, List[str]]], + """A list of dictionaries. in the form of: + Syntax: [{'Attribute Name':['Attribute description','Classification of attribute i.e. `Dimension' or `Measure`]}] + + Example: [{'Customer ID': ['Unique identifier', 'Dimension']},{'Sales Amount': ['Total sales value', 'Measure']}] + """, + ], + ) -> str: + log.info(f"--- Starting save_data_product Tool for '{data_product_name}'---") + filename = "attributes.csv" + + if not isinstance(attribute_data, list): + error_msg = "Error: Input data must be a list." + log.error(error_msg) + return error_msg + + processed_data = [] + headers = [ + "Data Product Name", + "Data Product Description", + "Attribute Name", + "Attribute Description", + "Attribute Classification", + ] + + for item in attribute_data: + if not isinstance(item, dict) or len(item) != 1: + error_msg = f"Error: Item must be a dictionary with a single attribute. Found: {item}" + log.error(error_msg) + return error_msg + + try: + attribute_name, details = list(item.items())[0] + if not isinstance(details, list) or len(details) != 2: + error_msg = f"Error: Value for '{attribute_name}' must be a list of [description, classification]." + log.error(error_msg) + return error_msg + + description, classification = details + processed_data.append( + { + "Data Product Name": data_product_name, + "Data Product Description": data_product_description, + "Attribute Name": attribute_name, + "Attribute Description": description, + "Attribute Classification": classification, + } + ) + except Exception as e: + error_msg = f"An unexpected error occurred while processing item {item}: {e}" + log.error(error_msg, exc_info=True) + return error_msg + + try: + with open(filename, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=headers) + writer.writeheader() + writer.writerows(processed_data) + full_path = os.path.abspath(filename) + success_msg = f"Successfully created and saved attribute data to {full_path}. You can now EXIT" + log.info(success_msg) + return success_msg + except IOError as e: + error_msg = f"Error: Could not write to file {filename}. Check permissions or path. Reason: {e}" + log.error(error_msg, exc_info=True) + return error_msg + + +class DataProductBuilderAgentTools: + def __init__( + self, + retrieval_tool: ConceptualSearchRetrievers, + manifest: Manifest, + ): + self._retrieval_tool = retrieval_tool + self.manifest = manifest + self.column_logic_results = [] + self.table_description = None + + def list_tables( + self, + ) -> List[Dict[str, str]]: + """ + Retrieve a list of unique tables from the dataset along with their glossary and domain information. + Returns: + List[Dict[str, str]]: A list of dictionaries, each containing: + - 'table_name': The name of the table. + - 'table_glossary': A description of what the table contains. + - 'table_domain': The domain or subject area the table belongs to. + """ + if self.table_description is None: + self.table_description = fetch_table_with_description( + self.manifest + ).drop_duplicates() + return self.table_description.to_dict(orient="records") + + def column_logic_store( + self, + column_table_combined: Annotated[ + Optional[str | List[str]], + "column_table_combined (str or List[str]): One or more strings in the format 'column_name$$##$$table_name", + ], + attribute: Annotated[ + str, "Name of the attribute that was used to fetch the columns." + ], + attribute_description: Annotated[str, "Description about the attribute"], + logic: Annotated[ + Optional[str | List[str]], + "One or more logic expressions corresponding to the columns.", + ], + ) -> str: + """ + Stores logic used to derive an attribute from one or more column-table combinations. + + Args: + column_table_combined (str or List[str]): One or more strings in the format 'column_name$$##$$table_name'. + logic (str or List[str]): One or more logic expressions corresponding to the columns. + attribute (str): Name of the attribute that was used to fetch the columns. + attribute_description (str): Description about the attribute + + Returns: + str: A success or error message confirming what was stored or explaining the failure. + """ + if column_table_combined is None or logic is None: + self.column_logic_results.append( + {"table_name": None, "column_name": None, "logic": None} + ) + return "[Stored] No matching column found. Logic: None" + + if isinstance(column_table_combined, str): + column_table_combined = [column_table_combined] + if isinstance(logic, str): + logic = [logic] + + if len(column_table_combined) != len(logic): + return "[Error] Mismatch: number of column-table strings and logic expressions must match." + + stored_entries = [] + for ct_str, lg in zip(column_table_combined, logic): + try: + column_name, table_name = ct_str.split("$$##$$") + except ValueError: + return f"[Error] Invalid format in '{ct_str}' (expected 'column_name$$##$$table_name')" + + self.column_logic_results.append( + { + "table_name": table_name, + "column_name": column_name, + "logic": lg, + "attribute": attribute, + "attribute_description": attribute_description, + } + ) + stored_entries.append( + f"[Stored] Logic: {lg} | Column: '{column_name}' | Table: '{table_name}'" + ) + pd.DataFrame(self.column_logic_results).to_csv( + "column_logic_results.csv", index=False + ) + return "\n".join(stored_entries) + + def column_retriever( + self, + table_names: Annotated[ + List[str], "A list of table names to restrict the search to." + ], + attribute_name: Annotated[ + str, "The name of the attribute to search for (e.g., 'Count of Products')." + ], + attribute_description: Annotated[ + str, + "A description of what the attribute represents.", + ], + ) -> List[Dict[str, str]]: + """ + Retrieves relevant columns from the provided list of tables that semantically match the given attribute name and description. + + Args: + table_names (List[str]): A list of table names to restrict the search to. + attribute_name (str): The name of the attribute to search for (e.g., "Count of Products"). + attribute_description (str): A description of what the attribute represents. + + Returns: + List[Dict[str, str]]: A deduplicated list of dictionaries, each containing: + - 'table_name': The table the column belongs to + - 'column_name': The name of the relevant column + """ + retrieved_results = self._retrieval_tool.column_retriever( + attribute_name=attribute_name, attribute_description=attribute_description + ) + + if not retrieved_results: + return "No appropriate column was found." + + def filter_by_table(doc: Document) -> bool: + return doc.metadata.get("table") in table_names + + if table_names: + retrieved_results = list(filter(filter_by_table, retrieved_results)) + + return [ + { + "table_name": result.metadata["table"], + "column_name": result.metadata["column"], + "column_glossary": result.page_content, + } + for result in retrieved_results + ] + + def list_tools( + self, + ) -> List[StructuredTool]: + return [ + StructuredTool.from_function( + func=self.list_tables, + name="list_tables", + description=""" + Retrieve a list of unique tables from the dataset along with their glossary and domain information. + Returns: + List[Dict[str, str]]: A list of dictionaries, each containing: + - 'table_name': The name of the table. + - 'table_glossary': A description of what the table contains. + - 'table_domain': The domain or subject area the table belongs to. + """, + ), + StructuredTool.from_function( + func=self.column_logic_store, + name="column_logic_store", + description=""" + Stores logic used to derive an attribute from one or more column-table combinations. + Returns: + str: A success or error message confirming what was stored or explaining the failure. + """, + ), + StructuredTool.from_function( + func=self.column_retriever, + name="column_retriever", + description="""Retrieves relevant columns from the provided list of tables that semantically match the given attribute name and description. + Returns: + List[Dict[str, str]]: A deduplicated list of dictionaries, each containing: + - 'table_name': The table the column belongs to + - 'column_name': The name of the relevant column + - 'column_glossary': The business description of the column + """, + ), + ] diff --git a/src/intugle/core/conceptual_search/agent/tools/web_tools.py b/src/intugle/core/conceptual_search/agent/tools/web_tools.py new file mode 100644 index 0000000..f03f227 --- /dev/null +++ b/src/intugle/core/conceptual_search/agent/tools/web_tools.py @@ -0,0 +1,35 @@ +import logging + +from langchain.schema import Document +from langchain_community.tools.tavily_search import TavilySearchResults +from langchain_core.tools import tool + +from intugle.core import settings + +log = logging.getLogger(__name__) + +web_search_tool = TavilySearchResults(k=3, tavily_api_key=settings.TAVILY_API_KEY) + + +@tool +def web_search(question: str) -> str: + """ + Google search to get more ideas on dimensions and measures used in the data product + + Args: + question (str): The question to web search for + + Returns: + web_results (str): results of the web search + """ + # Web search + log.info(f"Input statement: '{question}'") + try: + docs = web_search_tool.invoke({"query": question}) + web_results = "\n".join([d["content"] for d in docs]) + log.info(f"Web search results:\n{web_results}") # Print the search results + web_results = Document(page_content=web_results) + return web_results + except Exception as e: + log.error(f"Error during web search: {e}") + return Document(page_content=f"Error during web search: {e}") diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py index 90c384d..9d9a5f0 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py @@ -162,6 +162,7 @@ def remove_unwanted_concepts(concepts: list, filter_out: list): return list(filter(lambda concept: concept not in filter_out, concepts)) + def extract_concepts_column(text, llm): """ Extract key concepts from a column description. diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py index 144f946..6b03238 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py @@ -191,7 +191,7 @@ def prepare_chunk_document(manifest): table_description_df = fetch_table_with_description(manifest) - def build_table_description(row:pd.Series): + def build_table_description(row: pd.Series): description = [] @@ -201,26 +201,26 @@ def build_table_description(row:pd.Series): description.append(f"Table Domain: {domain.strip()}") - description += [f"Table Name: {row['table_name']}",f"Table Description: {row['table_description']}"] + description += [f"Table Name: {row['table_name']}", f"Table Description: {row['table_description']}"] return "\n".join(description) - table_description_df['table_description'] = table_description_df.apply(build_table_description,axis=1) + table_description_df['table_description'] = table_description_df.apply(build_table_description, axis=1) table_description_df['id'] = table_description_df['table_name'] log.info("[*] Extracting concepts for tables") - for i,data in tqdm(enumerate(list(batched(table_description_df,30)),1)): + for i, data in tqdm(enumerate(list(batched(table_description_df, 30)), 1)): - concepts = data["table_description"].apply(extract_concepts_table,llm=llm) + concepts = data["table_description"].apply(extract_concepts_table, llm=llm) - table_description_df.loc[data.index,"concepts"] = concepts + table_description_df.loc[data.index, "concepts"] = concepts log.info(f"[*] Batch {i} completed") table_description_df["concepts"] = table_description_df.concepts.apply(clean_concepts) - table_description_df.to_csv(table_description_path,index=False) + table_description_df.to_csv(table_description_path, index=False) else: @@ -228,9 +228,9 @@ def build_table_description(row:pd.Series): table_description_df["concepts"] = table_description_df.concepts.apply(ast.literal_eval) - def remove_concepts(concepts:list,to_remove=["hvac industry"]): + def remove_concepts(concepts: list, to_remove=["hvac industry"]): - return list(filter(lambda concept:concept.strip().lower() not in to_remove,concepts)) + return list(filter(lambda concept: concept.strip().lower() not in to_remove, concepts)) table_description_df["concepts"] = table_description_df.concepts.apply(remove_concepts) @@ -238,6 +238,6 @@ def remove_concepts(concepts:list,to_remove=["hvac industry"]): for index, row in table_description_df.iterrows(): - data_documents.append(VDocument(page_content=row['table_description'].strip().lower(), metadata={"source": row['id'], "row": index,"concepts":row['concepts']})) + data_documents.append(VDocument(page_content=row['table_description'].strip().lower(), metadata={"source": row['id'], "row": index, "concepts": row['concepts']})) return data_documents \ No newline at end of file diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py new file mode 100644 index 0000000..c4e4827 --- /dev/null +++ b/src/intugle/core/conceptual_search/search.py @@ -0,0 +1,114 @@ +import logging +import os + +import pandas as pd + +from langchain_community.callbacks import get_openai_callback + +from intugle.core import settings +from intugle.core.conceptual_search.agent.initializer import ( + data_product_builder_agent, + data_product_planner_agent, +) +from intugle.core.conceptual_search.agent.retrievers import ConceptualSearchRetrievers +from intugle.core.conceptual_search.agent.tools.tool_builder import ( + DataProductBuilderAgentTools, + DataProductPlannerAgentTools, +) +from intugle.core.conceptual_search.graph_based_column_search.networkx_initializers import ( + prepare_networkx_graph as prepare_column_networkx_graph, +) +from intugle.core.conceptual_search.graph_based_table_search.networkx_initializers import ( + prepare_networkx_graph as prepare_table_networkx_graph, +) +from intugle.core.conceptual_search.utils import batched +from intugle.core.llms.chat import ChatModelLLM +from intugle.parser.manifest import Manifest, ManifestLoader + +log = logging.getLogger(__name__) + + +class ConceptualSearch: + def __init__(self): + log.info("Initializing ConceptualSearch...") + self.manifest = self._load_manifest() + self._initialize_graphs() + self.retriever = ConceptualSearchRetrievers() + + self.llm = ChatModelLLM.get_llm( + model_name=settings.LLM_PROVIDER, + llm_config={"temperature": 0.05}, + ) + + self._data_product_planner_tool = DataProductPlannerAgentTools( + retrieval_tool=self.retriever + ) + self._data_product_builder_tool = DataProductBuilderAgentTools( + retrieval_tool=self.retriever, manifest=self.manifest + ) + + self._data_product_planner_agent = data_product_planner_agent( + llm=self.llm, tools=self._data_product_planner_tool.list_tools() + ) + self._data_product_builder_agent = data_product_builder_agent( + llm=self.llm, tools=self._data_product_builder_tool.list_tools() + ) + + def _load_manifest(self) -> Manifest: + log.info(f"Loading manifest from project base: {settings.PROJECT_BASE}") + manifest_loader = ManifestLoader(settings.PROJECT_BASE) + manifest_loader.load() + return manifest_loader.manifest + + def _initialize_graphs(self): + log.info("Initializing conceptual search graphs...") + prepare_table_networkx_graph(self.manifest) + prepare_column_networkx_graph(self.manifest) + log.info("Conceptual search graphs initialized.") + + def generate_data_product(self, attributes_df: pd.DataFrame): + BATCH_SIZE = 2 + + if attributes_df.shape[0] <= 0: + raise ValueError("Empty data product plan") + + total_records = attributes_df.shape[0] + log.info(f"πŸš€ Starting processing of {total_records} attributes...") + + cost = 0 + for b in batched(attributes_df, BATCH_SIZE): + messages = [ + { + "messages": [ + ( + "user", + f"attribute_name: {row['Attribute Name']} \n\n attribute_description: {row['Attribute Description']} \n\n attribute_type: {row['Attribute Classification']}", + ) + ] + } + for _, row in b.iterrows() + ] + + with get_openai_callback() as cb: + self._data_product_builder_agent.batch(messages) + cost += cb.total_cost + + dp = pd.read_csv("column_logic_results.csv") + dp["source"] = dp["table_name"] + "$$##$$" + dp["column_name"] + return dp, cost + + def generate_data_product_plan(self, query: str, additional_context: str = None): + if additional_context and additional_context.strip(): + query += f"\nAdditional Context:\n{additional_context}" + + total_cost = 0.0 + with get_openai_callback() as cb: + self._data_product_planner_agent.invoke( + input={"messages": [("user", query)]}, + ) + total_cost = cb.total_cost + + if os.path.exists("attributes.csv"): + return pd.read_csv("attributes.csv"), total_cost + + return None, None diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index b367ae3..1a7b2cf 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -88,7 +88,7 @@ def fetch_table_with_description(manifest: "Manifest") -> pd.DataFrame: { "table_name": table.name, "table_description": table.description or "", - "domain_name": source.schema, # Using schema as domain for now + "domain_name": source.schema, # Using schema as domain for now } ) diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index 43ab181..df7991a 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -105,6 +105,7 @@ class Settings(BaseSettings): VECTOR_COLLECTION_NAME: str = os.getcwd().split('/')[-1] QDRANT_URL: str = "http://localhost:6333" QDRANT_API_KEY: Optional[str] = None + TAVILY_API_KEY: Optional[str] = None EMBEDDING_MODEL_NAME: str = "openai:ada" TOKENIZER_MODEL_NAME: str = "cl100k_base" From dbf3a39bcf2e4982e7f2c699c954704d2e6b1481 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Sat, 25 Oct 2025 18:18:54 +0530 Subject: [PATCH 09/26] needs testing --- .../conceptual_search/agent/retrievers.py | 17 ++++--- .../agent/tools/tool_builder.py | 12 ++--- src/intugle/core/conceptual_search/utils.py | 48 +++++++++++++++++++ src/intugle/core/llms/embeddings.py | 1 + uv.lock | 16 +++++++ 5 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/intugle/core/conceptual_search/agent/retrievers.py b/src/intugle/core/conceptual_search/agent/retrievers.py index 6e93f0a..1629957 100644 --- a/src/intugle/core/conceptual_search/agent/retrievers.py +++ b/src/intugle/core/conceptual_search/agent/retrievers.py @@ -53,20 +53,19 @@ def data_products_retriever( ) return [] - def table_retriever(self, query: str) -> list[Document]: - results = asyncio.run(self.table_graph.get_shortlisted_tables(query)) + async def table_retriever(self, query: str) -> list[Document]: + results = await self.table_graph.get_shortlisted_tables(query) return self.to_documents(results) - def column_retriever( + async def column_retriever( self, attribute_name: str, attribute_description: str = "" ) -> list[Document]: - results_attribute_name = asyncio.run( - self.column_graph.get_shortlisted_columns(query=attribute_name) - ) - - results_attribue_description = asyncio.run( - self.column_graph.get_shortlisted_columns(query=attribute_description) + # Run column searches concurrently + results = await asyncio.gather( + self.column_graph.get_shortlisted_columns(query=attribute_name), + self.column_graph.get_shortlisted_columns(query=attribute_description), ) + results_attribute_name, results_attribue_description = results if results_attribute_name.empty and results_attribue_description.empty: return [] diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py index 88fb359..0aa1a8d 100644 --- a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -49,7 +49,7 @@ def list_tools( ), StructuredTool.from_function( name="retrieve_table_details", - func=self.retrieve_table_details, + coroutine=self.retrieve_table_details, description=""" Retrieve table details from a database based on a natural language statement. @@ -106,7 +106,7 @@ def retrieve_existing_data_products( log.error(f"ERROR: An exception occurred during data product retrieval: {e}", exc_info=True) return [f"ERROR: An exception occurred during data product retrieval: {e}"] - def retrieve_table_details( + async def retrieve_table_details( self, statement: Annotated[ str, @@ -117,7 +117,7 @@ def retrieve_table_details( ) -> List[Dict[str, Any]]: log.info(f"--- Executing retrieve_table_details for statement: '{statement}' ---") try: - documents = self.retrieval_tool.table_retriever(statement) + documents = await self.retrieval_tool.table_retriever(statement) log.info(f"Retriever returned {len(documents)} potential document(s).") if not documents: @@ -299,7 +299,7 @@ def column_logic_store( ) return "\n".join(stored_entries) - def column_retriever( + async def column_retriever( self, table_names: Annotated[ List[str], "A list of table names to restrict the search to." @@ -325,7 +325,7 @@ def column_retriever( - 'table_name': The table the column belongs to - 'column_name': The name of the relevant column """ - retrieved_results = self._retrieval_tool.column_retriever( + retrieved_results = await self._retrieval_tool.column_retriever( attribute_name=attribute_name, attribute_description=attribute_description ) @@ -373,7 +373,7 @@ def list_tools( """, ), StructuredTool.from_function( - func=self.column_retriever, + coroutine=self.column_retriever, name="column_retriever", description="""Retrieves relevant columns from the provided list of tables that semantically match the given attribute name and description. Returns: diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index 1a7b2cf..571d4b4 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -133,3 +133,51 @@ def fetch_column_with_description(manifest: "Manifest") -> pd.DataFrame: ) return pd.DataFrame(column_data) + + +def extract_data_product_info(documents): + extracted_info = [] + + if not documents: + print("No documents found.") + return extracted_info # Return empty list if no documents are present + + for doc in documents: + data_product_name = None + dimensions = [] + measures = [] + + # Extracting Data Product Name from page_content + # if "Data_Product:" in doc.page_content: + data_product_name = doc.page_content.split("Data_Product:")[-1].strip() + + # Extracting Dimensions and Measures from metadata + dimensions = doc.metadata.get("Dimensions", "").split(", ") + measures = doc.metadata.get("Measures", "").split(", ") + + # Append extracted information as a dictionary + extracted_info.append( + { + "Data_Product": data_product_name, + "Dimensions": dimensions, + "Measures": measures, + } + ) + + return extracted_info + + +def extract_table_details(documents): + extracted_info = [] + if not documents: + print("No tables found.") + return extracted_info # Return empty list if no documents are present + + for doc in documents: + # Extracting table details + table_details = doc.page_content.strip("text: ") + + # Append extracted information as a dictionary + extracted_info.append({doc.metadata["table"]: table_details}) + + return extracted_info diff --git a/src/intugle/core/llms/embeddings.py b/src/intugle/core/llms/embeddings.py index 30ba081..8fb3117 100644 --- a/src/intugle/core/llms/embeddings.py +++ b/src/intugle/core/llms/embeddings.py @@ -145,3 +145,4 @@ async def aencode(self, documents: Documents, embeddings_types: ListEmbeddingsTy print("Error: ", e) import traceback print(traceback.format_exc()) + raise e diff --git a/uv.lock b/uv.lock index 03ccc62..afb794d 100644 --- a/uv.lock +++ b/uv.lock @@ -1710,6 +1710,7 @@ dependencies = [ { name = "rich" }, { name = "scikit-learn" }, { name = "symspellpy" }, + { name = "tavily-python" }, { name = "trieregex" }, { name = "xgboost" }, ] @@ -1782,6 +1783,7 @@ requires-dist = [ { name = "snowflake-snowpark-python", extras = ["pandas"], marker = "extra == 'snowflake'", specifier = ">=1.12.0" }, { name = "sqlglot", marker = "extra == 'databricks'", specifier = ">=27.20.0" }, { name = "symspellpy", specifier = ">=6.9.0" }, + { name = "tavily-python", specifier = ">=0.1.11" }, { name = "trieregex", specifier = ">=1.0.0" }, { name = "xgboost", specifier = ">=3.0.4" }, ] @@ -5166,6 +5168,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] +[[package]] +name = "tavily-python" +version = "0.7.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "requests" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/42/ce2329635b844dda548110a5dfa0ab5631cdc1085e15c2d68b1850a2d112/tavily_python-0.7.12.tar.gz", hash = "sha256:661945bbc9284cdfbe70fb50de3951fd656bfd72e38e352481d333a36ae91f5a", size = 17282, upload-time = "2025-09-10T17:02:01.281Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/e2/dbc246d9fb24433f77b17d9ee4e750a1e2718432ebde2756589c9154cbad/tavily_python-0.7.12-py3-none-any.whl", hash = "sha256:00d09b9de3ca02ef9a994cf4e7ae43d4ec9d199f0566ba6e52cbfcbd07349bd1", size = 15473, upload-time = "2025-09-10T17:01:59.859Z" }, +] + [[package]] name = "tenacity" version = "9.1.2" From febc33868d31d45ec7a2def5bec97c739fd3b02e Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 27 Oct 2025 12:51:41 +0530 Subject: [PATCH 10/26] added notebook compatitibility for create graphs. implemeted retriever for data product builder --- .../conceptual_search/agent/retrievers.py | 85 +++++++++++++++++-- .../agent/tools/tool_builder.py | 6 +- .../networkx_initializers.py | 7 +- .../networkx_initializers.py | 7 +- 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/src/intugle/core/conceptual_search/agent/retrievers.py b/src/intugle/core/conceptual_search/agent/retrievers.py index 1629957..dd86368 100644 --- a/src/intugle/core/conceptual_search/agent/retrievers.py +++ b/src/intugle/core/conceptual_search/agent/retrievers.py @@ -4,16 +4,26 @@ import pandas as pd from langchain_core.documents import Document +from qdrant_client import models +from intugle.core import settings from intugle.core.conceptual_search.graph_based_column_search.retreiver import ( GraphSearch as ColumnGraphSearch, ) from intugle.core.conceptual_search.graph_based_table_search.retreiver import ( GraphSearch as TableGraphSearch, ) +from intugle.core.llms.embeddings import Embeddings, EmbeddingsType +from intugle.core.vector_store import AsyncQdrantService +from intugle.core.vector_store.qdrant import ( + QdrantVectorConfiguration, + VectorSearchKwargs, +) log = logging.getLogger(__name__) +DATA_PRODUCTS_COLLECTION_NAME = "data_products" + class ConceptualSearchRetrievers: def to_documents(self, results: pd.DataFrame) -> list[Document]: @@ -39,19 +49,82 @@ def to_documents(self, results: pd.DataFrame) -> list[Document]: def __init__(self): self.table_graph = TableGraphSearch() self.column_graph = ColumnGraphSearch() + self.embedding_model = Embeddings( + model_name=settings.EMBEDDING_MODEL_NAME, + tokenizer_model=settings.TOKENIZER_MODEL_NAME, + ) - def data_products_retriever( + async def data_products_retriever( self, query: str, ) -> list[Document]: """ - Retrieves existing data products. - NOTE: This is a placeholder and currently returns no documents. + Retrieves existing data products from the vector store. """ - log.warning( - f"Attempted to retrieve data products for query: '{query}'. This feature is not yet implemented." + log.info(f"Retrieving data products for query: '{query}'") + + # 1. Vectorize query + vectors = await self.embedding_model.aencode( + [query], embeddings_types=[EmbeddingsType.DENSE] ) - return [] + query_vector = vectors[EmbeddingsType.DENSE][0] + vector_name = f"{self.embedding_model.model_name}-{EmbeddingsType.DENSE}" + + # 2. Configure Qdrant + vectors_config = { + vector_name: models.VectorParams( + size=self.embedding_model.embeddings_size, + distance=models.Distance.COSINE, + ) + } + configuration = QdrantVectorConfiguration(vectors_config=vectors_config) + + # 3. Search Qdrant + async with AsyncQdrantService( + collection_name=DATA_PRODUCTS_COLLECTION_NAME, + collection_configurations=configuration, + ) as qdb: + try: + results = await qdb.search( + query=query_vector, + search_using=vector_name, + includes=["metadata"], + search_kwargs=dict( + VectorSearchKwargs( + search_type="similarity", + score_threshold=0.5, + top_k=2, + search_params=models.SearchParams( + hnsw_ef=128, + exact=False, + ), + ) + ), + ) + except Exception as e: + log.error( + f"Failed to search data products collection '{DATA_PRODUCTS_COLLECTION_NAME}': {e}", + exc_info=True, + ) + return [] + + if not results or not results.points: + log.info("No data products found for the query.") + return [] + + # 4. Format results + formatted_results = [ + Document( + page_content=res.payload.get("content", ""), + metadata={ + "Dimensions": res.payload.get("Dimensions"), + "Measures": res.payload.get("Measures"), + }, + ) + for res in results.points + ] + log.info(f"Found {len(formatted_results)} data products.") + return formatted_results async def table_retriever(self, query: str) -> list[Document]: results = await self.table_graph.get_shortlisted_tables(query) diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py index 0aa1a8d..8ca984a 100644 --- a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -32,7 +32,7 @@ def list_tools( return [ StructuredTool.from_function( name="retrieve_existing_data_products", - func=self.retrieve_existing_data_products, + coroutine=self.retrieve_existing_data_products, description="""Retrieve dimensions and measures for similar existing data products. This function uses a retriever (retriever_dp) to fetch documents describing @@ -82,7 +82,7 @@ def list_tools( ), ] - def retrieve_existing_data_products( + async def retrieve_existing_data_products( self, statement: Annotated[ str, @@ -92,7 +92,7 @@ def retrieve_existing_data_products( ) -> List[Dict[str, Any]]: log.info(f"--- Executing retrieve_existing_data_products for statement: '{statement}' ---") try: - documents = self.retrieval_tool.data_products_retriever(statement) + documents = await self.retrieval_tool.data_products_retriever(statement) log.info(f"Retriever returned {len(documents)} potential document(s).") if not documents: diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py index 6510f4b..881ae36 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py @@ -3,6 +3,8 @@ import os import pickle +from concurrent.futures import ThreadPoolExecutor + import networkx as nx from intugle.core import settings @@ -38,7 +40,10 @@ def build_knowledge_graph(doc): log.info(f"doc length: {len(doc)}") print("*" * 100) - embeddings = asyncio.run(create_embeddings(doc)) + # Run asyncio in a separate thread to avoid "event loop is already running" error in notebooks + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(asyncio.run, create_embeddings(doc)) + embeddings = future.result() # Add nodes to the graph print("Adding nodes to the graph...") diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py index 59d921f..ace5d78 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py @@ -3,6 +3,8 @@ import os import pickle +from concurrent.futures import ThreadPoolExecutor + import networkx as nx from intugle.core import settings @@ -25,7 +27,10 @@ def build_knowledge_graph(doc): graph = nx.Graph() log.info("Creating embeddings for table chunks...") - embeddings = asyncio.run(create_embeddings(doc)) + # Run asyncio in a separate thread to avoid "event loop is already running" error in notebooks + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(asyncio.run, create_embeddings(doc)) + embeddings = future.result() log.info("Adding nodes to the graph...") for i, chunk in enumerate(doc): From ba2b68dfdd1f02b50df5d83d737524e0b1dac050 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 27 Oct 2025 15:17:42 +0530 Subject: [PATCH 11/26] added langgraph for dev --- .../networkx_initializers.py | 4 +- .../networkx_initializers.py | 4 +- src/intugle/core/conceptual_search/search.py | 50 ++++++--- src/intugle/core/conceptual_search/utils.py | 29 +++++ uv.lock | 105 +++++++++++++++++- 5 files changed, 167 insertions(+), 25 deletions(-) diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py index 881ae36..73f3cc4 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py @@ -96,12 +96,12 @@ def build_knowledge_graph(doc): return graph, embeddings -def prepare_networkx_graph(manifest): +def prepare_networkx_graph(manifest, force_recreate=False): graph_path = os.path.join(settings.GRAPH_DIR, GraphFileName.FIELD) - if os.path.exists(graph_path): + if os.path.exists(graph_path) and not force_recreate: print('[!] Column search graph already build ... skipping the step') return diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py index ace5d78..5850a78 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py @@ -63,11 +63,11 @@ def build_knowledge_graph(doc): return graph, embeddings -def prepare_networkx_graph(manifest): +def prepare_networkx_graph(manifest, force_recreate=False): graph_path = os.path.join(settings.GRAPH_DIR, GraphFileName.TABLE) - if os.path.exists(graph_path): + if os.path.exists(graph_path) and not force_recreate: print('[!] Table search graph already built... skipping the step') return diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py index c4e4827..6713fba 100644 --- a/src/intugle/core/conceptual_search/search.py +++ b/src/intugle/core/conceptual_search/search.py @@ -3,7 +3,7 @@ import pandas as pd -from langchain_community.callbacks import get_openai_callback +from langchain_core.runnables import chain from intugle.core import settings from intugle.core.conceptual_search.agent.initializer import ( @@ -21,7 +21,10 @@ from intugle.core.conceptual_search.graph_based_table_search.networkx_initializers import ( prepare_networkx_graph as prepare_table_networkx_graph, ) -from intugle.core.conceptual_search.utils import batched +from intugle.core.conceptual_search.utils import ( + batched, + langfuse_callback_handler, +) from intugle.core.llms.chat import ChatModelLLM from intugle.parser.manifest import Manifest, ManifestLoader @@ -29,10 +32,10 @@ class ConceptualSearch: - def __init__(self): + def __init__(self, force_recreate=False): log.info("Initializing ConceptualSearch...") self.manifest = self._load_manifest() - self._initialize_graphs() + self._initialize_graphs(force_recreate=force_recreate) self.retriever = ConceptualSearchRetrievers() self.llm = ChatModelLLM.get_llm( @@ -53,6 +56,7 @@ def __init__(self): self._data_product_builder_agent = data_product_builder_agent( llm=self.llm, tools=self._data_product_builder_tool.list_tools() ) + self.callbacks = [langfuse_callback_handler()] def _load_manifest(self) -> Manifest: log.info(f"Loading manifest from project base: {settings.PROJECT_BASE}") @@ -60,13 +64,13 @@ def _load_manifest(self) -> Manifest: manifest_loader.load() return manifest_loader.manifest - def _initialize_graphs(self): + def _initialize_graphs(self, force_recreate=False): log.info("Initializing conceptual search graphs...") - prepare_table_networkx_graph(self.manifest) - prepare_column_networkx_graph(self.manifest) + prepare_table_networkx_graph(self.manifest, force_recreate) + prepare_column_networkx_graph(self.manifest, force_recreate) log.info("Conceptual search graphs initialized.") - def generate_data_product(self, attributes_df: pd.DataFrame): + async def generate_data_product(self, attributes_df: pd.DataFrame): BATCH_SIZE = 2 if attributes_df.shape[0] <= 0: @@ -89,9 +93,17 @@ def generate_data_product(self, attributes_df: pd.DataFrame): for _, row in b.iterrows() ] - with get_openai_callback() as cb: - self._data_product_builder_agent.batch(messages) - cost += cb.total_cost + @chain + async def run(inputs: dict): + await self._data_product_builder_agent.abatch(inputs["messages"]) + + await run.ainvoke( + {"messages": messages}, + config={ + "callbacks": self.callbacks, + "run_name": "Data Product Building", + }, + ) dp = pd.read_csv("column_logic_results.csv") dp["source"] = dp["table_name"] + "$$##$$" + dp["column_name"] @@ -101,14 +113,16 @@ def generate_data_product_plan(self, query: str, additional_context: str = None) if additional_context and additional_context.strip(): query += f"\nAdditional Context:\n{additional_context}" - total_cost = 0.0 - with get_openai_callback() as cb: - self._data_product_planner_agent.invoke( - input={"messages": [("user", query)]}, - ) - total_cost = cb.total_cost + self._data_product_planner_agent.invoke( + input={"messages": [("user", query)]}, + config={ + "callbacks": self.callbacks, + "metadata": {"Query": query}, + "run_name": "Data product planning", + }, + ) if os.path.exists("attributes.csv"): - return pd.read_csv("attributes.csv"), total_cost + return pd.read_csv("attributes.csv").head(5) return None, None diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index 571d4b4..a94690b 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -1,5 +1,6 @@ import json import logging +import os import re from typing import TYPE_CHECKING, Any @@ -181,3 +182,31 @@ def extract_table_details(documents): extracted_info.append({doc.metadata["table"]: table_details}) return extracted_info + + +def langfuse_callback_handler(): + try: + from langfuse.callback import CallbackHandler + except ImportError: + log.info( + "[!] langfuse package not installed. Please install it to use langfuse callback handler." + ) + return None + + langfuse_handler = CallbackHandler( + public_key=os.environ["LANGFUSE_PUBLIC_KEY"], + secret_key=os.environ["LANGFUSE_SECRET_KEY"], + host=os.environ["LANGFUSE_HOST"], + environment=os.environ.get("LANGFUSE_ENVIRONMENT_NAME", "dev"), + # trace_name=trace_name, + # session_id=session_id, + # tags=tags + ) + + # Check for langfuse handler then only add it to the callback + try: + langfuse_handler.auth_check() + return langfuse_handler + except Exception as ex: + log.info(f"[!] Could not connect to langfuse: {str(ex)}") + return None diff --git a/uv.lock b/uv.lock index afb794d..593cd29 100644 --- a/uv.lock +++ b/uv.lock @@ -259,6 +259,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "backports-asyncio-runner" version = "1.2.0" @@ -1729,6 +1738,7 @@ snowflake = [ dev = [ { name = "databricks-sql-connector" }, { name = "ipykernel" }, + { name = "langfuse" }, { name = "pysonar" }, { name = "pyspark" }, { name = "pytest" }, @@ -1793,6 +1803,7 @@ provides-extras = ["snowflake", "databricks"] dev = [ { name = "databricks-sql-connector", specifier = ">=4.1.3" }, { name = "ipykernel", specifier = ">=6.30.1" }, + { name = "langfuse", specifier = "==2.60.4" }, { name = "pysonar", specifier = ">=1.2.0.2419" }, { name = "pyspark", specifier = ">=4.0.1" }, { name = "pytest", specifier = ">=8.4.1" }, @@ -2423,6 +2434,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/0d/41a51b40d24ff0384ec4f7ab8dd3dcea8353c05c973836b5e289f1465d4f/langchain_text_splitters-0.3.11-py3-none-any.whl", hash = "sha256:cf079131166a487f1372c8ab5d0bfaa6c0a4291733d9c43a34a16ac9bcd6a393", size = 33845, upload-time = "2025-08-31T23:02:57.195Z" }, ] +[[package]] +name = "langfuse" +version = "2.60.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "backoff" }, + { name = "httpx" }, + { name = "idna" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/fc/3cbf173d8a4caf595d13de3f7ed659b4199ae898e8f35fa03167fa1c527a/langfuse-2.60.4.tar.gz", hash = "sha256:1fb6f1228dd6ed8ce49ffeba92eae1209bd46f0bd67fc0c6a5ec881af60ea68a", size = 152779, upload-time = "2025-05-06T13:30:17.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/3f/4c404b5d8c767ab8a64fa1ec2ac2c13c94c5e442cab336dff1dbe12d56d3/langfuse-2.60.4-py3-none-any.whl", hash = "sha256:458e685d9c924addca6a268ab01369e32777e6514694325a6dd1578a0c1e6fa6", size = 275355, upload-time = "2025-05-06T13:30:15.173Z" }, +] + [[package]] name = "langgraph" version = "0.6.8" @@ -3275,11 +3305,11 @@ wheels = [ [[package]] name = "packaging" -version = "25.0" +version = "24.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950, upload-time = "2024-11-08T09:47:47.202Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451, upload-time = "2024-11-08T09:47:44.722Z" }, ] [[package]] @@ -5672,6 +5702,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" }, ] +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/23/bb82321b86411eb51e5a5db3fb8f8032fd30bd7c2d74bfe936136b2fa1d6/wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04", size = 53482, upload-time = "2025-08-12T05:51:44.467Z" }, + { url = "https://files.pythonhosted.org/packages/45/69/f3c47642b79485a30a59c63f6d739ed779fb4cc8323205d047d741d55220/wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2", size = 38676, upload-time = "2025-08-12T05:51:32.636Z" }, + { url = "https://files.pythonhosted.org/packages/d1/71/e7e7f5670c1eafd9e990438e69d8fb46fa91a50785332e06b560c869454f/wrapt-1.17.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd341868a4b6714a5962c1af0bd44f7c404ef78720c7de4892901e540417111c", size = 38957, upload-time = "2025-08-12T05:51:54.655Z" }, + { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" }, + { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" }, + { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" }, + { url = "https://files.pythonhosted.org/packages/f8/83/ed6baf89ba3a56694700139698cf703aac9f0f9eb03dab92f57551bd5385/wrapt-1.17.3-cp310-cp310-win32.whl", hash = "sha256:a36692b8491d30a8c75f1dfee65bef119d6f39ea84ee04d9f9311f83c5ad9390", size = 36464, upload-time = "2025-08-12T05:53:01.204Z" }, + { url = "https://files.pythonhosted.org/packages/2f/90/ee61d36862340ad7e9d15a02529df6b948676b9a5829fd5e16640156627d/wrapt-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:afd964fd43b10c12213574db492cb8f73b2f0826c8df07a68288f8f19af2ebe6", size = 38748, upload-time = "2025-08-12T05:53:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c3/cefe0bd330d389c9983ced15d326f45373f4073c9f4a8c2f99b50bfea329/wrapt-1.17.3-cp310-cp310-win_arm64.whl", hash = "sha256:af338aa93554be859173c39c85243970dc6a289fa907402289eeae7543e1ae18", size = 36810, upload-time = "2025-08-12T05:52:51.906Z" }, + { url = "https://files.pythonhosted.org/packages/52/db/00e2a219213856074a213503fdac0511203dceefff26e1daa15250cc01a0/wrapt-1.17.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:273a736c4645e63ac582c60a56b0acb529ef07f78e08dc6bfadf6a46b19c0da7", size = 53482, upload-time = "2025-08-12T05:51:45.79Z" }, + { url = "https://files.pythonhosted.org/packages/5e/30/ca3c4a5eba478408572096fe9ce36e6e915994dd26a4e9e98b4f729c06d9/wrapt-1.17.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5531d911795e3f935a9c23eb1c8c03c211661a5060aab167065896bbf62a5f85", size = 38674, upload-time = "2025-08-12T05:51:34.629Z" }, + { url = "https://files.pythonhosted.org/packages/31/25/3e8cc2c46b5329c5957cec959cb76a10718e1a513309c31399a4dad07eb3/wrapt-1.17.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0610b46293c59a3adbae3dee552b648b984176f8562ee0dba099a56cfbe4df1f", size = 38959, upload-time = "2025-08-12T05:51:56.074Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" }, + { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/bb/59/e0adfc831674a65694f18ea6dc821f9fcb9ec82c2ce7e3d73a88ba2e8718/wrapt-1.17.3-cp311-cp311-win32.whl", hash = "sha256:c31eebe420a9a5d2887b13000b043ff6ca27c452a9a22fa71f35f118e8d4bf89", size = 36457, upload-time = "2025-08-12T05:53:03.936Z" }, + { url = "https://files.pythonhosted.org/packages/83/88/16b7231ba49861b6f75fc309b11012ede4d6b0a9c90969d9e0db8d991aeb/wrapt-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:0b1831115c97f0663cb77aa27d381237e73ad4f721391a9bfb2fe8bc25fa6e77", size = 38745, upload-time = "2025-08-12T05:53:02.885Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1e/c4d4f3398ec073012c51d1c8d87f715f56765444e1a4b11e5180577b7e6e/wrapt-1.17.3-cp311-cp311-win_arm64.whl", hash = "sha256:5a7b3c1ee8265eb4c8f1b7d29943f195c00673f5ab60c192eba2d4a7eae5f46a", size = 36806, upload-time = "2025-08-12T05:52:53.368Z" }, + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", size = 54003, upload-time = "2025-08-12T05:51:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", size = 39025, upload-time = "2025-08-12T05:51:37.156Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", size = 39108, upload-time = "2025-08-12T05:51:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, + { url = "https://files.pythonhosted.org/packages/e8/cf/7d848740203c7b4b27eb55dbfede11aca974a51c3d894f6cc4b865f42f58/wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", size = 36711, upload-time = "2025-08-12T05:53:10.074Z" }, + { url = "https://files.pythonhosted.org/packages/57/54/35a84d0a4d23ea675994104e667ceff49227ce473ba6a59ba2c84f250b74/wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", size = 38885, upload-time = "2025-08-12T05:53:08.695Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/66e54407c59d7b02a3c4e0af3783168fff8e5d61def52cda8728439d86bc/wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", size = 36896, upload-time = "2025-08-12T05:52:55.34Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/cd864b2a14f20d14f4c496fab97802001560f9f41554eef6df201cd7f76c/wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", size = 54132, upload-time = "2025-08-12T05:51:49.864Z" }, + { url = "https://files.pythonhosted.org/packages/d5/46/d011725b0c89e853dc44cceb738a307cde5d240d023d6d40a82d1b4e1182/wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", size = 39091, upload-time = "2025-08-12T05:51:38.935Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9e/3ad852d77c35aae7ddebdbc3b6d35ec8013af7d7dddad0ad911f3d891dae/wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", size = 39172, upload-time = "2025-08-12T05:51:59.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, + { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/da8560695e9284810b8d3df8a19396a6e40e7518059584a1a394a2b35e0a/wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", size = 37178, upload-time = "2025-08-12T05:53:12.605Z" }, + { url = "https://files.pythonhosted.org/packages/db/c8/b71eeb192c440d67a5a0449aaee2310a1a1e8eca41676046f99ed2487e9f/wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", size = 39310, upload-time = "2025-08-12T05:53:11.106Z" }, + { url = "https://files.pythonhosted.org/packages/45/20/2cda20fd4865fa40f86f6c46ed37a2a8356a7a2fde0773269311f2af56c7/wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", size = 37266, upload-time = "2025-08-12T05:52:56.531Z" }, + { url = "https://files.pythonhosted.org/packages/77/ed/dd5cf21aec36c80443c6f900449260b80e2a65cf963668eaef3b9accce36/wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", size = 56544, upload-time = "2025-08-12T05:51:51.109Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/450c651cc753877ad100c7949ab4d2e2ecc4d97157e00fa8f45df682456a/wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", size = 40283, upload-time = "2025-08-12T05:51:39.912Z" }, + { url = "https://files.pythonhosted.org/packages/d1/86/2fcad95994d9b572db57632acb6f900695a648c3e063f2cd344b3f5c5a37/wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", size = 40366, upload-time = "2025-08-12T05:52:00.693Z" }, + { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, + { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, + { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, + { url = "https://files.pythonhosted.org/packages/be/44/a1bd64b723d13bb151d6cc91b986146a1952385e0392a78567e12149c7b4/wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", size = 38717, upload-time = "2025-08-12T05:53:15.214Z" }, + { url = "https://files.pythonhosted.org/packages/79/d9/7cfd5a312760ac4dd8bf0184a6ee9e43c33e47f3dadc303032ce012b8fa3/wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", size = 41334, upload-time = "2025-08-12T05:53:14.178Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/10ad9781128ed2f99dbc474f43283b13fea8ba58723e98844367531c18e9/wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", size = 38471, upload-time = "2025-08-12T05:52:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + [[package]] name = "xgboost" version = "3.0.5" From 0dfe2b9ab60c3fa5c89b928f308c1eae5b8f021f Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 27 Oct 2025 17:02:49 +0530 Subject: [PATCH 12/26] added force recreate for qdrant --- pyproject.toml | 1 + .../graph_based_column_search/networkx_initializers.py | 2 +- .../graph_based_table_search/networkx_initializers.py | 2 +- src/intugle/core/conceptual_search/search.py | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5ecfc63..59bb5fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ lint = ["ruff"] dev = [ "databricks-sql-connector>=4.1.3", "ipykernel>=6.30.1", + "langfuse==2.60.4", "pysonar>=1.2.0.2419", "pyspark>=4.0.1", "pytest>=8.4.1", diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py index 73f3cc4..4cf3b4a 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/networkx_initializers.py @@ -42,7 +42,7 @@ def build_knowledge_graph(doc): # Run asyncio in a separate thread to avoid "event loop is already running" error in notebooks with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(asyncio.run, create_embeddings(doc)) + future = executor.submit(asyncio.run, create_embeddings(doc, recreate=True)) embeddings = future.result() # Add nodes to the graph diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py index 5850a78..7e31b6b 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/networkx_initializers.py @@ -29,7 +29,7 @@ def build_knowledge_graph(doc): log.info("Creating embeddings for table chunks...") # Run asyncio in a separate thread to avoid "event loop is already running" error in notebooks with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(asyncio.run, create_embeddings(doc)) + future = executor.submit(asyncio.run, create_embeddings(doc, recreate=True)) embeddings = future.result() log.info("Adding nodes to the graph...") diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py index 6713fba..8b061c3 100644 --- a/src/intugle/core/conceptual_search/search.py +++ b/src/intugle/core/conceptual_search/search.py @@ -109,11 +109,11 @@ async def run(inputs: dict): dp["source"] = dp["table_name"] + "$$##$$" + dp["column_name"] return dp, cost - def generate_data_product_plan(self, query: str, additional_context: str = None): + async def generate_data_product_plan(self, query: str, additional_context: str = None): if additional_context and additional_context.strip(): query += f"\nAdditional Context:\n{additional_context}" - self._data_product_planner_agent.invoke( + await self._data_product_planner_agent.ainvoke( input={"messages": [("user", query)]}, config={ "callbacks": self.callbacks, From 8bf9c43339476614f71be633370c7de626353a21 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 27 Oct 2025 17:55:30 +0530 Subject: [PATCH 13/26] Added project id --- .../conceptual_search/agent/retrievers.py | 7 ++- .../graph_based_column_search/utils.py | 7 ++- .../graph_based_table_search/utils.py | 7 ++- src/intugle/core/exceptions.py | 6 ++ src/intugle/core/project.py | 57 +++++++++++++++++++ src/intugle/core/settings.py | 10 +++- 6 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 src/intugle/core/exceptions.py create mode 100644 src/intugle/core/project.py diff --git a/src/intugle/core/conceptual_search/agent/retrievers.py b/src/intugle/core/conceptual_search/agent/retrievers.py index dd86368..ee8ac67 100644 --- a/src/intugle/core/conceptual_search/agent/retrievers.py +++ b/src/intugle/core/conceptual_search/agent/retrievers.py @@ -22,7 +22,12 @@ log = logging.getLogger(__name__) -DATA_PRODUCTS_COLLECTION_NAME = "data_products" + +def data_products_collection_name(): + return f"{settings.PROJECT_ID}_data_products" + + +DATA_PRODUCTS_COLLECTION_NAME = data_products_collection_name() class ConceptualSearchRetrievers: diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py index 9d9a5f0..16a8411 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/utils.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/utils.py @@ -28,7 +28,12 @@ tqdm.pandas() -CONCEPTUAL_SEARCH_COLLECTION_NAME = "conceptual_search_columns" + +def conceptual_search_collection_name(): + return f"{settings.PROJECT_ID}_conceptual_search_columns" + + +CONCEPTUAL_SEARCH_COLLECTION_NAME = conceptual_search_collection_name() def collection_name(prefix: str, suffix: str = QdrantCollectionSuffix.FIELD): diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py index 6b03238..4aeb1f4 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py @@ -28,7 +28,12 @@ tqdm.pandas() -CONCEPTUAL_SEARCH_COLLECTION_NAME = "conceptual_search_tables" + +def conceptual_search_collection_name(): + return f"{settings.PROJECT_ID}_conceptual_search_tables" + + +CONCEPTUAL_SEARCH_COLLECTION_NAME = conceptual_search_collection_name() def collection_name(prefix: str, suffix: str = QdrantCollectionSuffix.TABLE): diff --git a/src/intugle/core/exceptions.py b/src/intugle/core/exceptions.py new file mode 100644 index 0000000..6fa818b --- /dev/null +++ b/src/intugle/core/exceptions.py @@ -0,0 +1,6 @@ +class IntugleException(Exception): + """Base exception for all intugle errors.""" + + def __init__(self, message: str): + self.message = message + super().__init__(self.message) diff --git a/src/intugle/core/project.py b/src/intugle/core/project.py new file mode 100644 index 0000000..d068ac2 --- /dev/null +++ b/src/intugle/core/project.py @@ -0,0 +1,57 @@ +import logging +import uuid + +from pathlib import Path + +import yaml + +from intugle.core.exceptions import IntugleException + +log = logging.getLogger(__name__) + +CONFIG_FILE_NAME = "config.yaml" + + +class Project: + """ + Manages the project configuration, specifically the project ID. + """ + + def __init__(self, project_base: str): + self.project_base = Path(project_base) + self.config_path = self.project_base / CONFIG_FILE_NAME + self._project_id = None + self._load_or_create_config() + + def _load_or_create_config(self): + """Loads the project config or creates it if it doesn't exist.""" + try: + self.project_base.mkdir(parents=True, exist_ok=True) + if self.config_path.exists(): + with open(self.config_path, "r") as f: + config = yaml.safe_load(f) + self._project_id = config.get("project_id") + if not self._project_id: + self._generate_and_save_project_id() + else: + self._generate_and_save_project_id() + except (IOError, yaml.YAMLError) as e: + raise IntugleException(f"Error loading or creating config file at {self.config_path}: {e}") from e + + def _generate_and_save_project_id(self): + """Generates a new project ID and saves it to the config file.""" + self._project_id = str(uuid.uuid4()) + config = {"project_id": self._project_id} + try: + with open(self.config_path, "w") as f: + yaml.dump(config, f) + log.info(f"Generated and saved new project ID: {self._project_id}") + except IOError as e: + raise IntugleException(f"Error saving config file at {self.config_path}: {e}") from e + + @property + def project_id(self) -> str: + """Returns the project ID.""" + if not self._project_id: + raise IntugleException("Project ID not loaded.") + return self._project_id diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index df7991a..ac636b5 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -9,6 +9,7 @@ from dotenv import load_dotenv from pydantic_settings import BaseSettings, SettingsConfigDict +from intugle.core.project import Project from intugle.core.utilities.configs import load_model_configuration, load_profiles_configuration load_dotenv(dotenv_path=Path(__file__).resolve().parent.parent / ".env") @@ -29,7 +30,9 @@ class Settings(BaseSettings): """Global Configuration""" UPSTREAM_SAMPLE_LIMIT: int = 10 - MODEL_DIR_PATH: str = str(Path(os.path.split(os.path.abspath(__file__))[0]).parent.joinpath("artifacts")) + MODEL_DIR_PATH: str = str( + Path(os.path.split(os.path.abspath(__file__))[0]).parent.joinpath("artifacts") + ) MODEL_RESULTS_PATH: str = os.path.join("model", "model_results") DI_CONFIG: dict = load_model_configuration("DI", {}) @@ -41,12 +44,13 @@ class Settings(BaseSettings): # DIRECTORY STRUCTURE ENVS PROJECT_BASE: str = create_project_base_if_not_exists() + PROJECT_ID: str = Project(PROJECT_BASE).project_id MODELS_DIR_NAME: str = "models" GRAPH_DIR_NAME: str = "kg" MODELS_DIR: str = os.path.join(PROJECT_BASE, MODELS_DIR_NAME) GRAPH_DIR: str = os.path.join(PROJECT_BASE, GRAPH_DIR_NAME) DESCRIPTIONS_DIR: str = os.path.join(PROJECT_BASE, "descriptions") - + PROFILES_PATH: str = os.path.join(os.getcwd(), "profiles.yml") PROFILES: dict = load_profiles_configuration(PROFILES_PATH) @@ -102,7 +106,7 @@ class Settings(BaseSettings): POSTGRES_SCHEMA: Optional[str] = "public" # Vector - VECTOR_COLLECTION_NAME: str = os.getcwd().split('/')[-1] + VECTOR_COLLECTION_NAME: str = os.getcwd().split("/")[-1] QDRANT_URL: str = "http://localhost:6333" QDRANT_API_KEY: Optional[str] = None TAVILY_API_KEY: Optional[str] = None From 0d635f8d645b216894bea3d46d1f5fb75b78f5b4 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 28 Oct 2025 00:08:42 +0530 Subject: [PATCH 14/26] removed domain, removed cost tracking --- .../conceptual_search/graph_based_table_search/utils.py | 6 ------ src/intugle/core/conceptual_search/search.py | 5 ++--- src/intugle/core/conceptual_search/utils.py | 3 +-- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py index 4aeb1f4..e3b19ad 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/utils.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/utils.py @@ -200,12 +200,6 @@ def build_table_description(row: pd.Series): description = [] - domain = row["domain_name"] - - if not pd.isna(domain) and len(domain.strip()) != 0: - - description.append(f"Table Domain: {domain.strip()}") - description += [f"Table Name: {row['table_name']}", f"Table Description: {row['table_description']}"] return "\n".join(description) diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py index 8b061c3..e3ef2e7 100644 --- a/src/intugle/core/conceptual_search/search.py +++ b/src/intugle/core/conceptual_search/search.py @@ -79,7 +79,6 @@ async def generate_data_product(self, attributes_df: pd.DataFrame): total_records = attributes_df.shape[0] log.info(f"πŸš€ Starting processing of {total_records} attributes...") - cost = 0 for b in batched(attributes_df, BATCH_SIZE): messages = [ { @@ -107,7 +106,7 @@ async def run(inputs: dict): dp = pd.read_csv("column_logic_results.csv") dp["source"] = dp["table_name"] + "$$##$$" + dp["column_name"] - return dp, cost + return dp async def generate_data_product_plan(self, query: str, additional_context: str = None): if additional_context and additional_context.strip(): @@ -123,6 +122,6 @@ async def generate_data_product_plan(self, query: str, additional_context: str = ) if os.path.exists("attributes.csv"): - return pd.read_csv("attributes.csv").head(5) + return pd.read_csv("attributes.csv") return None, None diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index a94690b..b619bce 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -89,12 +89,11 @@ def fetch_table_with_description(manifest: "Manifest") -> pd.DataFrame: { "table_name": table.name, "table_description": table.description or "", - "domain_name": source.schema, # Using schema as domain for now } ) if not table_data: - return pd.DataFrame(columns=["table_name", "table_description", "domain_name"]) + return pd.DataFrame(columns=["table_name", "table_description"]) return pd.DataFrame(table_data) From 7ae9fb4f7516f70ac7896c7d55df8f17432bf30a Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 28 Oct 2025 00:47:04 +0530 Subject: [PATCH 15/26] corrected tool outputs --- .../core/conceptual_search/agent/tools/tool_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py index 8ca984a..83a6bef 100644 --- a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -97,14 +97,14 @@ async def retrieve_existing_data_products( if not documents: log.info("No similar data product documents were found by the retriever.") - return ["No similar data product documents were found by the retriever."] + return [] result = extract_data_product_info(documents) log.info(f"Extraction process resulted in {len(result)} structured data product detail(s).") return result except Exception as e: log.error(f"ERROR: An exception occurred during data product retrieval: {e}", exc_info=True) - return [f"ERROR: An exception occurred during data product retrieval: {e}"] + raise e async def retrieve_table_details( self, From ff30dff2c5cf625eae27b4d9550c3e07140e623a Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 3 Nov 2025 15:03:26 +0530 Subject: [PATCH 16/26] dataproduct planner APIs --- src/intugle/core/conceptual_search/plan.py | 120 +++++++++++++++++++ src/intugle/core/conceptual_search/search.py | 34 ++++-- 2 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 src/intugle/core/conceptual_search/plan.py diff --git a/src/intugle/core/conceptual_search/plan.py b/src/intugle/core/conceptual_search/plan.py new file mode 100644 index 0000000..7e209d3 --- /dev/null +++ b/src/intugle/core/conceptual_search/plan.py @@ -0,0 +1,120 @@ +import pandas as pd +from typing import List + +class DataProductPlan: + """ + A helper class to manage and modify a data product plan DataFrame. + """ + def __init__(self, df: pd.DataFrame): + if not isinstance(df, pd.DataFrame): + raise TypeError("df must be a pandas DataFrame.") + self._df = df.copy() + if '_is_active' not in self._df.columns: + self._df['_is_active'] = True + + @property + def name(self) -> str: + """Gets the name of the data product.""" + return self._df['Data Product Name'].iloc[0] + + @name.setter + def name(self, new_name: str): + """Sets the name of the data product.""" + self._df['Data Product Name'] = new_name + + @property + def description(self) -> str: + """Gets the description of the data product.""" + return self._df['Data Product Description'].iloc[0] + + @description.setter + def description(self, new_description: str): + """Sets the description of the data product.""" + self._df['Data Product Description'] = new_description + + def rename_attribute(self, old_name: str, new_name: str): + """Renames an attribute.""" + if old_name not in self._df['Attribute Name'].values: + raise ValueError(f"Attribute '{old_name}' not found in the plan.") + if new_name in self._df['Attribute Name'].values: + raise ValueError(f"Attribute '{new_name}' already exists in the plan.") + self._df.loc[self._df['Attribute Name'] == old_name, 'Attribute Name'] = new_name + + def set_attribute_description(self, attribute_name: str, new_description: str): + """Sets the description for a specific attribute.""" + if attribute_name not in self._df['Attribute Name'].values: + raise ValueError(f"Attribute '{attribute_name}' not found in the plan.") + self._df.loc[self._df['Attribute Name'] == attribute_name, 'Attribute Description'] = new_description + + def set_attribute_classification(self, attribute_name: str, new_classification: str): + """Sets the classification for a specific attribute.""" + if new_classification not in ['Dimension', 'Measure']: + raise ValueError("Classification must be either 'Dimension' or 'Measure'.") + if attribute_name not in self._df['Attribute Name'].values: + raise ValueError(f"Attribute '{attribute_name}' not found in the plan.") + self._df.loc[self._df['Attribute Name'] == attribute_name, 'Attribute Classification'] = new_classification + + def disable_attribute(self, attribute_name: str): + """Disables an attribute from the final output.""" + if attribute_name not in self._df['Attribute Name'].values: + raise ValueError(f"Attribute '{attribute_name}' not found in the plan.") + self._df.loc[self._df['Attribute Name'] == attribute_name, '_is_active'] = False + + def enable_attribute(self, attribute_name: str): + """Enables an attribute for the final output.""" + if attribute_name not in self._df['Attribute Name'].values: + raise ValueError(f"Attribute '{attribute_name}' not found in the plan.") + self._df.loc[self._df['Attribute Name'] == attribute_name, '_is_active'] = True + + def to_df(self) -> pd.DataFrame: + """Returns the final DataFrame with active attributes.""" + return self._df[self._df['_is_active']].drop(columns=['_is_active']) + + def __str__(self) -> str: + """Provides a string representation of the data product plan.""" + if self._df.empty: + return "Data Product Plan (empty)" + + name = self.name + description = self.description + + attributes_str = "" + for _, row in self._df.iterrows(): + status = "" if row['_is_active'] else " (Disabled)" + attributes_str += f" - {row['Attribute Name']}{status} ({row['Attribute Classification']}): {row['Attribute Description']}\n" + + return ( + f"Data Product: {name}\n" + f"Description: {description}\n" + f"Attributes:\n{attributes_str}" + ) + + def _repr_html_(self) -> str: + """Provides a rich HTML representation for Jupyter notebooks.""" + if self._df.empty: + return "
Data Product Plan (empty)
" + + name = self.name + description = self.description + + html = f"
" + html += f"

Data Product: {name}

" + html += f"

Description: {description}

" + html += f"

Attributes:

" + html += f" " + html += f" " + for _, row in self._df.iterrows(): + status = "Active" if row['_is_active'] else "Disabled" + html += f" " + html += f" " + html += f" " + html += f" " + html += f" " + html += f" " + html += f"
Attribute NameClassificationDescriptionStatus
{row['Attribute Name']}{row['Attribute Classification']}{row['Attribute Description']}{status}
" + html += f"
" + return html + + def display(self): + """Prints the string representation of the plan.""" + print(str(self)) \ No newline at end of file diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py index e3ef2e7..fd987fb 100644 --- a/src/intugle/core/conceptual_search/search.py +++ b/src/intugle/core/conceptual_search/search.py @@ -1,5 +1,6 @@ import logging import os +from typing import Union import pandas as pd @@ -21,6 +22,7 @@ from intugle.core.conceptual_search.graph_based_table_search.networkx_initializers import ( prepare_networkx_graph as prepare_table_networkx_graph, ) +from intugle.core.conceptual_search.plan import DataProductPlan from intugle.core.conceptual_search.utils import ( batched, langfuse_callback_handler, @@ -56,7 +58,8 @@ def __init__(self, force_recreate=False): self._data_product_builder_agent = data_product_builder_agent( llm=self.llm, tools=self._data_product_builder_tool.list_tools() ) - self.callbacks = [langfuse_callback_handler()] + handler = langfuse_callback_handler() + self.callbacks = [handler] if handler else [] def _load_manifest(self) -> Manifest: log.info(f"Loading manifest from project base: {settings.PROJECT_BASE}") @@ -70,7 +73,16 @@ def _initialize_graphs(self, force_recreate=False): prepare_column_networkx_graph(self.manifest, force_recreate) log.info("Conceptual search graphs initialized.") - async def generate_data_product(self, attributes_df: pd.DataFrame): + async def generate_data_product( + self, plan: Union[DataProductPlan, pd.DataFrame] + ): + if isinstance(plan, DataProductPlan): + attributes_df = plan.to_df() + elif isinstance(plan, pd.DataFrame): + attributes_df = plan + else: + raise TypeError("plan must be either a DataProductPlan or a pandas DataFrame.") + BATCH_SIZE = 2 if attributes_df.shape[0] <= 0: @@ -108,20 +120,22 @@ async def run(inputs: dict): dp["source"] = dp["table_name"] + "$$##$$" + dp["column_name"] return dp - async def generate_data_product_plan(self, query: str, additional_context: str = None): + async def generate_data_product_plan( + self, query: str, additional_context: str = None + ) -> DataProductPlan | None: + if additional_context and additional_context.strip(): query += f"\nAdditional Context:\n{additional_context}" await self._data_product_planner_agent.ainvoke( input={"messages": [("user", query)]}, config={ - "callbacks": self.callbacks, - "metadata": {"Query": query}, - "run_name": "Data product planning", - }, + "callbacks": self.callbacks, + "metadata": {"Query": query}, + "run_name": "Data product planning", + }, ) if os.path.exists("attributes.csv"): - return pd.read_csv("attributes.csv") - - return None, None + return DataProductPlan(pd.read_csv("attributes.csv")) + return None From c58a12214d7426ac3ce2b2977256fe981e7f1ce0 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 3 Nov 2025 15:13:25 +0530 Subject: [PATCH 17/26] storing plan in memory --- .../conceptual_search/agent/tools/tool_builder.py | 3 +++ src/intugle/core/conceptual_search/search.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py index 83a6bef..2500123 100644 --- a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -25,6 +25,7 @@ class DataProductPlannerAgentTools: def __init__(self, retrieval_tool: ConceptualSearchRetrievers): self.retrieval_tool = retrieval_tool + self.generated_plan: Optional[pd.DataFrame] = None def list_tools( self, @@ -191,6 +192,8 @@ def save_data_product( log.error(error_msg, exc_info=True) return error_msg + self.generated_plan = pd.DataFrame(processed_data, columns=headers) + try: with open(filename, "w", newline="", encoding="utf-8") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headers) diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py index fd987fb..a902f28 100644 --- a/src/intugle/core/conceptual_search/search.py +++ b/src/intugle/core/conceptual_search/search.py @@ -121,9 +121,15 @@ async def run(inputs: dict): return dp async def generate_data_product_plan( - self, query: str, additional_context: str = None + self, query: str, additional_context: str = None, use_cache: bool = False ) -> DataProductPlan | None: - + if use_cache and os.path.exists("attributes.csv"): + log.info("Loading data product plan from attributes.csv (cache).") + return DataProductPlan(pd.read_csv("attributes.csv")) + + log.info("Generating new data product plan...") + self._data_product_planner_tool.generated_plan = None # Clear previous plan + if additional_context and additional_context.strip(): query += f"\nAdditional Context:\n{additional_context}" @@ -136,6 +142,7 @@ async def generate_data_product_plan( }, ) - if os.path.exists("attributes.csv"): - return DataProductPlan(pd.read_csv("attributes.csv")) + if self._data_product_planner_tool.generated_plan is not None: + return DataProductPlan(self._data_product_planner_tool.generated_plan) + return None From 03063c372b12dedeb42079b2e6200028d36c4333 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Mon, 3 Nov 2025 19:32:06 +0530 Subject: [PATCH 18/26] add etl model mapping --- .../core/conceptual_search/agent/prompts.py | 34 +++--- .../agent/tools/tool_builder.py | 112 +++++++++--------- src/intugle/core/conceptual_search/search.py | 80 +++++++++++-- src/intugle/core/conceptual_search/utils.py | 4 +- .../smart_query_generator/models/models.py | 11 ++ .../smart_query_generator/utils/helpers.py | 8 ++ 6 files changed, 162 insertions(+), 87 deletions(-) create mode 100644 src/intugle/libs/smart_query_generator/utils/helpers.py diff --git a/src/intugle/core/conceptual_search/agent/prompts.py b/src/intugle/core/conceptual_search/agent/prompts.py index dde6fe9..f670f16 100644 --- a/src/intugle/core/conceptual_search/agent/prompts.py +++ b/src/intugle/core/conceptual_search/agent/prompts.py @@ -60,22 +60,24 @@ ) -data_product_builder_prompt = """You are a data exploration assistant. You are tasked with identifying the correct column(s) across available data sources that satisfy the attribute definition: +data_product_builder_prompt = """You are a data exploration assistant. Your task is to identify the correct database column that satisfies a given attribute definition for a data product. You have access to the following tools: -list_of_tables: Retrieve a list of unique tables from the dataset along with their glossary and domain information. -column_logic_store: Stores logic used to derive an attribute from one or more column-table combinations. -column_retriever: Retrieves relevant columns from the provided list of tables that semantically match the given attribute name and description. - -Use the following strategy: -- Iteratively explore all available tables using the list_of_tables tool. -- For relevant tables, use column_retriever to examine column names and column descriptions. -- Continue the process until a column (or combination of columns) clearly maps to the given attribute description. -- If no single column is sufficient, determine a transformation logic using a combination of related columns. -- Do not stop until either: - -- The attribute is matched with high confidence. - -- All possibilities are exhausted and it's determined that the attribute cannot be constructed with available data. -- At the end, store the matched column(s) and their corresponding tables. The data transformation logic needed to compute the final measure or dimension (in SQL or pseudo-code). - -**Attribute Details*** +- `list_tables`: Retrieve a list of all available tables with their descriptions. +- `column_retriever`: Search for relevant columns within a list of tables based on the attribute's name and description. +- `column_logic_store`: Persist the final mapping of an attribute to a database column and its transformation logic. + +Follow this strategy for EACH attribute you are given: +1. Start by using `list_tables` to get an overview of the available data. +2. Based on the table descriptions, identify a few candidate tables that might contain the required data. +3. Use `column_retriever` with the list of candidate tables to find the most relevant columns. +4. Analyze the retrieved columns. If you find a direct match, you are ready to store it. +5. If the attribute is a **Measure** (e.g., 'Total Sales', 'Number of Customers'), you MUST determine the correct aggregation function (e.g., 'sum', 'count', 'average'). +6. Once you have identified the correct column and any necessary logic, you MUST call the `column_logic_store` tool to save the result. + - For a **Dimension** that maps directly to a column, provide the `attribute_name`, `attribute_description`, `attribute_classification` ('dimension'), and the `column_table_combined` string. + - For a **Measure**, you MUST provide the `attribute_name`, `attribute_description`, `attribute_classification` ('measure'), the `column_table_combined` string, and the appropriate `measure_func` (e.g., 'sum', 'count'). + - If no suitable column is found, call `column_logic_store` with `column_table_combined` set to None. + +**Attribute Details** +You will be provided with the attribute's name, description, and classification (Dimension or Measure). """ diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py index 2500123..8ca81e8 100644 --- a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -8,6 +8,7 @@ from langchain_core.documents import Document from langchain_core.tools import StructuredTool +from pydantic import BaseModel from intugle.core.conceptual_search.agent.retrievers import ( ConceptualSearchRetrievers, @@ -17,11 +18,25 @@ extract_table_details, fetch_table_with_description, ) +from intugle.libs.smart_query_generator.models.models import ( + CategoryType, + MeasureFunctionType, +) from intugle.parser.manifest import Manifest log = logging.getLogger(__name__) +class MappedAttribute(BaseModel): + attribute_name: str + attribute_description: str + attribute_classification: CategoryType + table_name: Optional[str] = None + column_name: Optional[str] = None + logic: Optional[str] = None + measure_func: Optional[MeasureFunctionType] = None + + class DataProductPlannerAgentTools: def __init__(self, retrieval_tool: ConceptualSearchRetrievers): self.retrieval_tool = retrieval_tool @@ -217,7 +232,7 @@ def __init__( ): self._retrieval_tool = retrieval_tool self.manifest = manifest - self.column_logic_results = [] + self.column_logic_results: List[MappedAttribute] = [] self.table_description = None def list_tables( @@ -239,68 +254,53 @@ def list_tables( def column_logic_store( self, - column_table_combined: Annotated[ - Optional[str | List[str]], - "column_table_combined (str or List[str]): One or more strings in the format 'column_name$$##$$table_name", + attribute_name: Annotated[str, "Name of the attribute that was mapped."], + attribute_description: Annotated[str, "Description of the attribute."], + attribute_classification: Annotated[ + CategoryType, + "Classification of the attribute as 'dimension' or 'measure'.", ], - attribute: Annotated[ - str, "Name of the attribute that was used to fetch the columns." + column_table_combined: Annotated[ + Optional[str], + "A single string in the format 'column_name$$##$$table_name'. Use this if a direct column mapping is found. Set to None if no mapping is found.", ], - attribute_description: Annotated[str, "Description about the attribute"], logic: Annotated[ - Optional[str | List[str]], - "One or more logic expressions corresponding to the columns.", + Optional[str], + "The transformation or aggregation logic (e.g., 'SUM', 'COUNT', or a SQL expression).", + ], + measure_func: Annotated[ + Optional[MeasureFunctionType], + "The aggregation function for a measure (e.g., 'sum', 'count'). Required if attribute_classification is 'measure'.", ], ) -> str: """ - Stores logic used to derive an attribute from one or more column-table combinations. - - Args: - column_table_combined (str or List[str]): One or more strings in the format 'column_name$$##$$table_name'. - logic (str or List[str]): One or more logic expressions corresponding to the columns. - attribute (str): Name of the attribute that was used to fetch the columns. - attribute_description (str): Description about the attribute - - Returns: - str: A success or error message confirming what was stored or explaining the failure. + Stores the derived logic for a single data product attribute. """ - if column_table_combined is None or logic is None: - self.column_logic_results.append( - {"table_name": None, "column_name": None, "logic": None} - ) - return "[Stored] No matching column found. Logic: None" - - if isinstance(column_table_combined, str): - column_table_combined = [column_table_combined] - if isinstance(logic, str): - logic = [logic] - - if len(column_table_combined) != len(logic): - return "[Error] Mismatch: number of column-table strings and logic expressions must match." - - stored_entries = [] - for ct_str, lg in zip(column_table_combined, logic): + table_name, column_name = None, None + if column_table_combined: try: - column_name, table_name = ct_str.split("$$##$$") + column_name, table_name = column_table_combined.split("$$##$$") except ValueError: - return f"[Error] Invalid format in '{ct_str}' (expected 'column_name$$##$$table_name')" - - self.column_logic_results.append( - { - "table_name": table_name, - "column_name": column_name, - "logic": lg, - "attribute": attribute, - "attribute_description": attribute_description, - } - ) - stored_entries.append( - f"[Stored] Logic: {lg} | Column: '{column_name}' | Table: '{table_name}'" - ) - pd.DataFrame(self.column_logic_results).to_csv( - "column_logic_results.csv", index=False + return f"[Error] Invalid format in '{column_table_combined}' (expected 'column_name$$##$$table_name')" + + if ( + attribute_classification == CategoryType.measure + and measure_func is None + ): + return "[Error] For a 'measure' attribute, 'measure_func' is required." + + mapped_attribute = MappedAttribute( + attribute_name=attribute_name, + attribute_description=attribute_description, + attribute_classification=attribute_classification, + table_name=table_name, + column_name=column_name, + logic=logic, + measure_func=measure_func, ) - return "\n".join(stored_entries) + self.column_logic_results.append(mapped_attribute) + + return f"[Stored] Successfully stored logic for attribute: {attribute_name}" async def column_retriever( self, @@ -369,11 +369,7 @@ def list_tools( StructuredTool.from_function( func=self.column_logic_store, name="column_logic_store", - description=""" - Stores logic used to derive an attribute from one or more column-table combinations. - Returns: - str: A success or error message confirming what was stored or explaining the failure. - """, + description="""Stores the derived logic for a single data product attribute.""", ), StructuredTool.from_function( coroutine=self.column_retriever, diff --git a/src/intugle/core/conceptual_search/search.py b/src/intugle/core/conceptual_search/search.py index a902f28..76bac91 100644 --- a/src/intugle/core/conceptual_search/search.py +++ b/src/intugle/core/conceptual_search/search.py @@ -1,6 +1,5 @@ import logging import os -from typing import Union import pandas as pd @@ -28,6 +27,11 @@ langfuse_callback_handler, ) from intugle.core.llms.chat import ChatModelLLM +from intugle.libs.smart_query_generator import ( + CategoryType, + ETLModel, + FieldsModel, +) from intugle.parser.manifest import Manifest, ManifestLoader log = logging.getLogger(__name__) @@ -74,12 +78,14 @@ def _initialize_graphs(self, force_recreate=False): log.info("Conceptual search graphs initialized.") async def generate_data_product( - self, plan: Union[DataProductPlan, pd.DataFrame] - ): + self, plan: DataProductPlan | pd.DataFrame + ) -> ETLModel: if isinstance(plan, DataProductPlan): attributes_df = plan.to_df() + product_name = plan.name elif isinstance(plan, pd.DataFrame): attributes_df = plan + product_name = attributes_df["Data Product Name"].iloc[0] else: raise TypeError("plan must be either a DataProductPlan or a pandas DataFrame.") @@ -88,8 +94,11 @@ async def generate_data_product( if attributes_df.shape[0] <= 0: raise ValueError("Empty data product plan") + # Clear previous results before starting + self._data_product_builder_tool.column_logic_results = [] + total_records = attributes_df.shape[0] - log.info(f"πŸš€ Starting processing of {total_records} attributes...") + log.info(f"Starting processing of {total_records} attributes...") for b in batched(attributes_df, BATCH_SIZE): messages = [ @@ -97,7 +106,7 @@ async def generate_data_product( "messages": [ ( "user", - f"attribute_name: {row['Attribute Name']} \n\n attribute_description: {row['Attribute Description']} \n\n attribute_type: {row['Attribute Classification']}", + f"attribute_name: {row['Attribute Name']} \n\n attribute_description: {row['Attribute Description']} \n\n attribute_classification: {row['Attribute Classification']}", ) ] } @@ -116,19 +125,66 @@ async def run(inputs: dict): }, ) - dp = pd.read_csv("column_logic_results.csv") - dp["source"] = dp["table_name"] + "$$##$$" + dp["column_name"] - return dp + # Construct ETLModel from in-memory results + fields = [] + mapped_attributes = self._data_product_builder_tool.column_logic_results + + # Create a lookup for all columns in the manifest to get their IDs + column_id_map = { + f"{source.table.name}.{column.name}": f"{source.table.name}.{column.name}" + for source in self.manifest.sources.values() + for column in source.table.columns + } + + for attr in mapped_attributes: + if not attr.table_name or not attr.column_name: + log.warning( + f"Could not map attribute '{attr.attribute_name}', skipping." + ) + continue + + column_id_key = f"{attr.table_name}.{attr.column_name}" + column_id = column_id_map.get(column_id_key) + + if not column_id: + log.warning( + f"Could not find ID for column '{column_id_key}' for attribute '{attr.attribute_name}', skipping." + ) + continue + + field = FieldsModel( + id=column_id, + name=attr.attribute_name, + category=attr.attribute_classification, + ) + + if attr.attribute_classification == CategoryType.measure: + if not attr.measure_func: + log.warning( + f"Measure '{attr.attribute_name}' is missing a measure_func, defaulting to 'count'." + ) + field.measure_func = "count" + else: + field.measure_func = attr.measure_func + + fields.append(field) + + return ETLModel(name=product_name, fields=fields) async def generate_data_product_plan( - self, query: str, additional_context: str = None, use_cache: bool = False + self, + query: str, + additional_context: str = None, + use_cache: bool = False, ) -> DataProductPlan | None: if use_cache and os.path.exists("attributes.csv"): log.info("Loading data product plan from attributes.csv (cache).") return DataProductPlan(pd.read_csv("attributes.csv")) log.info("Generating new data product plan...") - self._data_product_planner_tool.generated_plan = None # Clear previous plan + self._data_product_planner_tool.generated_plan = ( + None # Clear previous plan + ) if additional_context and additional_context.strip(): query += f"\nAdditional Context:\n{additional_context}" @@ -143,6 +199,8 @@ async def generate_data_product_plan( ) if self._data_product_planner_tool.generated_plan is not None: - return DataProductPlan(self._data_product_planner_tool.generated_plan) + return DataProductPlan( + self._data_product_planner_tool.generated_plan + ) return None diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index b619bce..c92333c 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -187,7 +187,7 @@ def langfuse_callback_handler(): try: from langfuse.callback import CallbackHandler except ImportError: - log.info( + log.warning( "[!] langfuse package not installed. Please install it to use langfuse callback handler." ) return None @@ -207,5 +207,5 @@ def langfuse_callback_handler(): langfuse_handler.auth_check() return langfuse_handler except Exception as ex: - log.info(f"[!] Could not connect to langfuse: {str(ex)}") + log.warning(f"[!] Could not connect to langfuse: {str(ex)}") return None diff --git a/src/intugle/libs/smart_query_generator/models/models.py b/src/intugle/libs/smart_query_generator/models/models.py index f029e45..d5c4cb0 100644 --- a/src/intugle/libs/smart_query_generator/models/models.py +++ b/src/intugle/libs/smart_query_generator/models/models.py @@ -3,6 +3,8 @@ from pydantic import BaseModel, field_validator +from intugle.libs.smart_query_generator.utils.helpers import normalize_column_name + class MeasureFunctionType(str, Enum): count = "count" @@ -107,6 +109,12 @@ class FieldsModel(BaseModel): sql_code: Optional[str] = None join_opt: Optional[JoinOpt] = None + @field_validator("name", mode="before") + def normalize_name(cls, v): + if isinstance(v, str): + return normalize_column_name(v) + return v + class RangeModel(BaseModel): id: int | str @@ -127,6 +135,9 @@ class ETLModel(BaseModel): cart_id: int = 0 join: Optional[dict] = None + def __str__(self): + return self.model_dump_json() + class FieldDetailsModel(BaseModel): id: int | str diff --git a/src/intugle/libs/smart_query_generator/utils/helpers.py b/src/intugle/libs/smart_query_generator/utils/helpers.py new file mode 100644 index 0000000..d94f47f --- /dev/null +++ b/src/intugle/libs/smart_query_generator/utils/helpers.py @@ -0,0 +1,8 @@ +import re + + +def normalize_column_name(name: str) -> str: + name = name.lower() + name = re.sub(r'[^a-z0-9]+', '_', name) + name = name.strip('_') + return name \ No newline at end of file From 45cee9ae505425cdca9e28ff2f9734123b06deb6 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 4 Nov 2025 00:40:54 +0530 Subject: [PATCH 19/26] added query cleaning --- .../core/conceptual_search/agent/tools/tool_builder.py | 2 +- .../graph_based_column_search/retreiver.py | 3 +++ .../conceptual_search/graph_based_table_search/retreiver.py | 3 +++ src/intugle/core/conceptual_search/utils.py | 6 ++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py index 8ca81e8..66c193a 100644 --- a/src/intugle/core/conceptual_search/agent/tools/tool_builder.py +++ b/src/intugle/core/conceptual_search/agent/tools/tool_builder.py @@ -85,7 +85,7 @@ def list_tools( self.save_data_product, name="save_data_product", description=""" - Processes a list of attribute dictionaries and saves them to a CSV file, + Processes a list of attribute dictionaries and saves them with detailed print statements for monitoring. Each dictionary in the input list should represent a single attribute, diff --git a/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py b/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py index 30d1052..610cbb4 100644 --- a/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py +++ b/src/intugle/core/conceptual_search/graph_based_column_search/retreiver.py @@ -14,6 +14,7 @@ CONCEPTUAL_SEARCH_COLLECTION_NAME, ) from intugle.core.conceptual_search.models import GraphFileName +from intugle.core.conceptual_search.utils import clean_query from intugle.core.llms.embeddings import Embeddings, EmbeddingsType from intugle.core.vector_store import AsyncQdrantService from intugle.core.vector_store.qdrant import VectorSearchKwargs @@ -144,6 +145,8 @@ async def get_shortlisted_columns(self, query: str): log.info("Graph loaded successfully") + query = clean_query(query) + _, traversal_path, search_result = await self.traverse_graph( query, self.graph, diff --git a/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py b/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py index dec86b0..447baaa 100644 --- a/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py +++ b/src/intugle/core/conceptual_search/graph_based_table_search/retreiver.py @@ -13,6 +13,7 @@ CONCEPTUAL_SEARCH_COLLECTION_NAME, ) from intugle.core.conceptual_search.models import GraphFileName +from intugle.core.conceptual_search.utils import clean_query from intugle.core.llms.embeddings import Embeddings, EmbeddingsType from intugle.core.vector_store import AsyncQdrantService from intugle.core.vector_store.qdrant import VectorSearchKwargs @@ -135,6 +136,8 @@ async def get_shortlisted_tables(self, query: str): log.info("Graph loaded successfully") + query = clean_query(query) + _, traversal_path, search_result = await self.traverse_graph( query, self.graph, diff --git a/src/intugle/core/conceptual_search/utils.py b/src/intugle/core/conceptual_search/utils.py index c92333c..2b18b89 100644 --- a/src/intugle/core/conceptual_search/utils.py +++ b/src/intugle/core/conceptual_search/utils.py @@ -14,6 +14,12 @@ log = logging.getLogger(__name__) +def clean_query(s: str) -> str: + s = s.lower() + s = ' '.join(s.split()) + return s + + def batched(data: Any, n): for index in range(0, len(data), n): yield data[index : index + n] From b57c9e0cd0a8cac5902b0ffaf33cc9815578378d Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 4 Nov 2025 15:38:15 +0530 Subject: [PATCH 20/26] add conceptual search api to data product --- src/intugle/conceptual_search/__init__.py | 0 src/intugle/conceptual_search/engine.py | 52 ---------------- src/intugle/conceptual_search/models.py | 30 ---------- src/intugle/data_product.py | 73 ++++++++++++++++++++++- 4 files changed, 71 insertions(+), 84 deletions(-) delete mode 100644 src/intugle/conceptual_search/__init__.py delete mode 100644 src/intugle/conceptual_search/engine.py delete mode 100644 src/intugle/conceptual_search/models.py diff --git a/src/intugle/conceptual_search/__init__.py b/src/intugle/conceptual_search/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/intugle/conceptual_search/engine.py b/src/intugle/conceptual_search/engine.py deleted file mode 100644 index c1d099d..0000000 --- a/src/intugle/conceptual_search/engine.py +++ /dev/null @@ -1,52 +0,0 @@ -# src/intugle/conceptual_search/engine.py -from intugle.core import settings -from intugle.parser.manifest import ManifestLoader - -from .models import ConceptualAttribute, ConceptualPlan, MappedAttribute, MappedPlan - - -class ConceptualSearch: - def __init__(self, project_base: str = settings.PROJECT_BASE): - """Initializes by loading the manifest to provide context for the agents.""" - self.manifest_loader = ManifestLoader(project_base) - self.manifest_loader.load() - self.manifest = self.manifest_loader.manifest - # Initialize your LLM clients for the agents here - - def create_plan(self, concept: str) -> ConceptualPlan: - """ - Agent 1: Takes a high-level concept and generates a conceptual plan. - """ - print(f"Agent 1: Generating conceptual plan for '{concept}'...") - # --- YOUR AGENT 1 LOGIC GOES HERE --- - # This logic will use an LLM to transform the 'concept' string - # into a list of ConceptualAttribute objects. - # For this example, we'll use mock data. - - mock_attributes = [ - ConceptualAttribute(name="Patient Name", description="The full name of the patient", logic="Concatenate first and last names"), - ConceptualAttribute(name="Total Claims", description="The total number of claims filed by the patient", logic="Count all claims associated with the patient ID") - ] - - plan = ConceptualPlan(attributes=mock_attributes) - print("Agent 1: Plan created successfully.") - return plan - - def map_plan_to_columns(self, plan: ConceptualPlan) -> MappedPlan: - """ - Agent 2: Takes a conceptual plan and maps it to physical columns. - """ - print("Agent 2: Mapping conceptual plan to semantic layer columns...") - # --- YOUR AGENT 2 LOGIC GOES HERE --- - # This logic uses an LLM, providing the conceptual plan and the - # manifest (self.manifest) as context to find the real column_ids. - # For this example, we'll use mock data. - - mapped_attributes = [ - MappedAttribute(name="Patient Name", description="The full name of the patient", logic="Concatenate first and last names", column_id="patients.first"), - MappedAttribute(name="Total Claims", description="The total number of claims filed by the patient", logic="Count all claims associated with the patient ID", column_id="claims.id", measure_func="COUNT") - ] - - mapped_plan = MappedPlan(attributes=mapped_attributes) - print("Agent 2: Mapping complete.") - return mapped_plan diff --git a/src/intugle/conceptual_search/models.py b/src/intugle/conceptual_search/models.py deleted file mode 100644 index 67d835e..0000000 --- a/src/intugle/conceptual_search/models.py +++ /dev/null @@ -1,30 +0,0 @@ -# src/intugle/conceptual_search/models.py -from typing import List, Optional - -from pydantic import BaseModel, Field - - -class ConceptualAttribute(BaseModel): - """An abstract attribute for the desired data product.""" - name: str = Field(description="A business-friendly name for the column, e.g., 'Patient Age'") - description: str = Field(description="A detailed explanation of what this attribute represents.") - logic: str = Field(description="A natural language description of the calculation, e.g., 'Calculate the age based on the date of birth'") - - -class ConceptualPlan(BaseModel): - """The conceptual plan generated by Agent 1, which can be reviewed by the user.""" - attributes: List[ConceptualAttribute] - - def to_dict(self): - return self.model_dump() - - -class MappedAttribute(ConceptualAttribute): - """A conceptual attribute that has been mapped to a physical column.""" - column_id: str = Field(description="The fully qualified column ID from the semantic layer, e.g., 'patients.birth_date'") - measure_func: Optional[str] = Field(default=None, description="The aggregation function if any, e.g., 'COUNT', 'SUM'") - - -class MappedPlan(BaseModel): - """The final plan after Agent 2 has mapped attributes to columns.""" - attributes: List[MappedAttribute] diff --git a/src/intugle/data_product.py b/src/intugle/data_product.py index 70cf928..61d48ac 100644 --- a/src/intugle/data_product.py +++ b/src/intugle/data_product.py @@ -1,11 +1,17 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from intugle.adapters.factory import AdapterFactory from intugle.analysis.models import DataSet from intugle.conceptual_search.models import MappedPlan from intugle.core import settings +from intugle.core.conceptual_search.plan import DataProductPlan +from intugle.core.conceptual_search.search import ConceptualSearch from intugle.libs.smart_query_generator import SmartQueryGenerator -from intugle.libs.smart_query_generator.models.models import ETLModel, FieldDetailsModel, LinkModel +from intugle.libs.smart_query_generator.models.models import ( + ETLModel, + FieldDetailsModel, + LinkModel, +) from intugle.libs.smart_query_generator.utils.join import Join from intugle.parser.manifest import ManifestLoader @@ -32,6 +38,69 @@ def __init__(self, models_dir_path: str = settings.MODELS_DIR): self.join = Join(self.links, selected_fields) self.load_all() + self._conceptual_search: Optional[ConceptualSearch] = None + + def _get_conceptual_search(self) -> ConceptualSearch: + """Initializes and returns the ConceptualSearch instance.""" + if self._conceptual_search is None: + self._conceptual_search = ConceptualSearch() + return self._conceptual_search + + async def plan( + self, + query: str, + additional_context: str = None, + use_cache: bool = False, + ) -> Optional[DataProductPlan]: + """ + Generates a data product plan from a natural language query. + + Args: + query: The natural language query describing the desired data product. + additional_context: Optional additional context to guide the planning. + use_cache: Whether to use a cached plan if available. + + Returns: + A DataProductPlan object that can be reviewed and modified, or None if planning fails. + """ + cs = self._get_conceptual_search() + plan = await cs.generate_data_product_plan( + query, additional_context=additional_context, use_cache=use_cache + ) + return plan + + async def create_etl_model_from_plan(self, plan: DataProductPlan) -> ETLModel: + """ + Generates an ETLModel from a DataProductPlan. + + This method converts the high-level plan into a detailed, executable + ETLModel that defines the data product's fields and transformations. + + Args: + plan: The DataProductPlan to convert. + + Returns: + The generated ETLModel. + """ + cs = self._get_conceptual_search() + etl_model = await cs.generate_data_product(plan) + return etl_model + + async def build_from_plan(self, plan: DataProductPlan) -> DataSet: + """ + Builds a data product from a DataProductPlan. + + This is a convenience method that first generates the ETLModel from the + plan and then immediately builds the data product. + + Args: + plan: The DataProductPlan to build. + + Returns: + A new DataSet object pointing to the materialized table. + """ + etl_model = await self.create_etl_model_from_plan(plan) + return self.build(etl=etl_model) @staticmethod def etl_from_mapped_plan(plan: MappedPlan, product_name: str) -> dict: From c25a2328426637d253ba71b92497aa9c1a346db0 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 4 Nov 2025 15:58:00 +0530 Subject: [PATCH 21/26] removed unused import in dp --- src/intugle/data_product.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/intugle/data_product.py b/src/intugle/data_product.py index 61d48ac..a9be7f8 100644 --- a/src/intugle/data_product.py +++ b/src/intugle/data_product.py @@ -2,7 +2,6 @@ from intugle.adapters.factory import AdapterFactory from intugle.analysis.models import DataSet -from intugle.conceptual_search.models import MappedPlan from intugle.core import settings from intugle.core.conceptual_search.plan import DataProductPlan from intugle.core.conceptual_search.search import ConceptualSearch @@ -102,25 +101,6 @@ async def build_from_plan(self, plan: DataProductPlan) -> DataSet: etl_model = await self.create_etl_model_from_plan(plan) return self.build(etl=etl_model) - @staticmethod - def etl_from_mapped_plan(plan: MappedPlan, product_name: str) -> dict: - """Creates an ETLModel dictionary from a mapped conceptual plan.""" - fields = [] - for attr in plan.attributes: - field = { - "id": attr.column_id, - "name": attr.name, - } - if attr.measure_func: - field["category"] = "measure" - field["measure_func"] = attr.measure_func.lower() - fields.append(field) - - return { - "name": product_name, - "fields": fields - } - def load_all(self): sources = self.manifest.sources for source in sources.values(): From 973c9ecfc730803fdb42d334501ad0bcd9517708 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 4 Nov 2025 17:05:31 +0530 Subject: [PATCH 22/26] conceptual search test cases and quickstart --- .gitignore | 4 + conceptual_search_quickstart.ipynb | 158 +++++++++++++++++++++++++++++ test_col_search.py | 24 +++++ test_table_search.py | 25 +++++ 4 files changed, 211 insertions(+) create mode 100644 conceptual_search_quickstart.ipynb create mode 100644 test_col_search.py create mode 100644 test_table_search.py diff --git a/.gitignore b/.gitignore index ac0d473..f315400 100644 --- a/.gitignore +++ b/.gitignore @@ -223,3 +223,7 @@ profiles.yml profiles.yaml /intugle/ +.mcpregistry_github_token +.mcpregistry_registry_token + +/.gemini/ \ No newline at end of file diff --git a/conceptual_search_quickstart.ipynb b/conceptual_search_quickstart.ipynb new file mode 100644 index 0000000..5f3dc86 --- /dev/null +++ b/conceptual_search_quickstart.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conceptual Search Quickstart\n", + "\n", + "This notebook demonstrates how to use the new **Conceptual Search** functionality in the `intugle` library. The process involves two main steps:\n", + "\n", + "1. **Generate a Data Product Plan:** You provide a high-level goal in natural language (e.g., \"I want a report on patient allergies\"), and the Planner Agent generates a structured CSV file (`attributes.csv`) detailing the required dimensions and measures.\n", + "2. **Build the Data Product:** The Builder Agent takes the generated plan and maps each attribute to the most relevant columns in your data sources, creating a final mapping file (`column_logic_results.csv`).\n", + "\n", + "**Prerequisite:** This workflow requires a built `SemanticModel`, as the agents rely on the manifest YAML files it generates." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 1: Build the Semantic Model\n", + "\n", + "First, we need to build a `SemanticModel` from our data sources. This will profile the data, discover relationships, and generate the manifest files that the Conceptual Search agents need to understand your data landscape." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dotenv import load_dotenv\n", + "load_dotenv(override=True)\n", + "\n", + "from intugle import SemanticModel\n", + "\n", + "# Define the paths to your datasets\n", + "# For this example, we'll use the sample healthcare data included in the repo.\n", + "datasets = {\n", + " \"patients\": {\"path\": \"../sample_data/healthcare/patients.csv\", \"type\": \"csv\"},\n", + " \"allergies\": {\"path\": \"../sample_data/healthcare/allergies.csv\", \"type\": \"csv\"},\n", + " \"encounters\": {\"path\": \"../sample_data/healthcare/encounters.csv\", \"type\": \"csv\"},\n", + " \"conditions\": {\"path\": \"../sample_data/healthcare/conditions.csv\", \"type\": \"csv\"}\n", + "}\n", + "\n", + "# Build the semantic model\n", + "# This will generate the necessary YAML files in your project root directory.\n", + "sm = SemanticModel(datasets, domain=\"Healthcare\")\n", + "sm.build()\n", + "\n", + "print(\"Semantic Model has been successfully built.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2: Generate a Data Product Plan\n", + "\n", + "Now that the semantic model is built, we can use the `ConceptualSearch` class to generate a plan for our desired data product. We'll define our goal in a simple, natural language query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from intugle import DataProduct\n", + "\n", + "data_product = DataProduct()\n", + "\n", + "\n", + "query = \"patient 360\"\n", + "\n", + "\n", + "data_product_plan = await data_product.plan(query, \"patient 360 view\")\n", + "\n", + "# Display the plan. This is a special object that can be modified in code.\n", + "data_product_plan\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# data_product_plan.disable_attribute(\"Date of Birth\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3: Build the Data Product (Map Plan to Columns)\n", + "\n", + "With the plan generated, we can now pass it to the Builder Agent. This agent will intelligently search the semantic model to find the best table and column for each attribute in the plan." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# import pandas as pd\n", + "# from intugle.core.conceptual_search.search import ConceptualSearch\n", + "# conceptual_search = ConceptualSearch(force_recreate=False)\n", + "\n", + "\n", + "# plan_df = pd.read_csv(\"attributes.csv\")\n", + "\n", + "# The 'plan_df' is the DataFrame we generated in the previous step\n", + "if data_product_plan is not None:\n", + " etl_model = await data_product.create_etl_model_from_plan(data_product_plan)\n", + "\n", + "print(etl_model.model_dump_json(indent=2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result_dataset = data_product.build(etl_model)\n", + "\n", + "print(\"-- Generated SQL Query --\")\n", + "print(result_dataset.sql_query)\n", + "\n", + "print(\"\\n-- Resulting Data (First 5 Rows) --\")\n", + "result_dataset.to_df().head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "intugle", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test_col_search.py b/test_col_search.py new file mode 100644 index 0000000..4a82546 --- /dev/null +++ b/test_col_search.py @@ -0,0 +1,24 @@ + +# import asyncio +import asyncio + +from intugle.core import settings +from intugle.core.conceptual_search.graph_based_column_search.networkx_initializers import prepare_networkx_graph +from intugle.core.conceptual_search.graph_based_column_search.retreiver import GraphSearch +from intugle.parser.manifest import ManifestLoader + +manifest_loader = ManifestLoader(settings.MODELS_DIR) +manifest_loader.load() +manifest = manifest_loader.manifest + +prepare_networkx_graph(manifest) + +graph = GraphSearch() + +while True: + print("\n\nEnter query: ") + query = input() + if query == "q": + break + results = asyncio.run(graph.get_shortlisted_columns(query=query)) + print(results.sort_values(by="score", ascending=False)) diff --git a/test_table_search.py b/test_table_search.py new file mode 100644 index 0000000..bd7d931 --- /dev/null +++ b/test_table_search.py @@ -0,0 +1,25 @@ + +# import asyncio +import asyncio + +from intugle.core import settings +from intugle.core.conceptual_search.graph_based_table_search.networkx_initializers import prepare_networkx_graph +from intugle.core.conceptual_search.graph_based_table_search.retreiver import GraphSearch +from intugle.parser.manifest import ManifestLoader + +manifest_loader = ManifestLoader(settings.MODELS_DIR) +manifest_loader.load() +manifest = manifest_loader.manifest + +prepare_networkx_graph(manifest) + +graph = GraphSearch() + +while True: + print("\n\nEnter query: ") + query = input() + if query == "q": + break + results = asyncio.run(graph.get_shortlisted_tables(query=query)) + print(results.sort_values(by="score", ascending=False)) + # breakpoint() \ No newline at end of file From 4ac4dee3b698f507036a6ef03a6dc816ea502d53 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 11 Nov 2025 13:46:53 +0530 Subject: [PATCH 23/26] added tavily key conditional handling. added set project base function. --- .../conceptual_search/agent/initializer.py | 3 +- .../core/conceptual_search/agent/prompts.py | 4 +- .../agent/tools/web_tools.py | 5 +- src/intugle/core/settings.py | 8 +- src/intugle/streamlit_app/main.py | 4 +- uv.lock | 341 +++++++++++++++++- 6 files changed, 350 insertions(+), 15 deletions(-) diff --git a/src/intugle/core/conceptual_search/agent/initializer.py b/src/intugle/core/conceptual_search/agent/initializer.py index 84fa4a1..f6ccd47 100644 --- a/src/intugle/core/conceptual_search/agent/initializer.py +++ b/src/intugle/core/conceptual_search/agent/initializer.py @@ -4,6 +4,7 @@ from langchain_core.tools import StructuredTool from langgraph.prebuilt import create_react_agent +from intugle.core import settings from intugle.core.conceptual_search.agent.prompts import ( data_product_builder_prompt, data_product_planner_prompt, @@ -13,7 +14,7 @@ def data_product_planner_agent(llm, tools: List[StructuredTool]) -> RunnableSerializable: agent = create_react_agent( - llm, tools + [web_search], prompt=data_product_planner_prompt + llm, tools + ([web_search] if settings.TAVILY_API_KEY else []), prompt=data_product_planner_prompt ) return agent diff --git a/src/intugle/core/conceptual_search/agent/prompts.py b/src/intugle/core/conceptual_search/agent/prompts.py index f670f16..1198c4d 100644 --- a/src/intugle/core/conceptual_search/agent/prompts.py +++ b/src/intugle/core/conceptual_search/agent/prompts.py @@ -1,5 +1,7 @@ from langchain_core.prompts import ChatPromptTemplate +from intugle.core import settings + data_product_planner_prompt = ChatPromptTemplate.from_messages( [ ( @@ -8,7 +10,7 @@ "Follow these steps:\n" "1. Use `retrieve_existing_data_products(statement)` to identify similar data products and gather candidate Dimensions and Measures.\n" "2. Use `retrieve_table_details(statement)` to get relevant database tables. If the initial query does not return all required information, do not hesitate to run multiple queries with refined or different search terms (e.g., 'customer data', 'order transactions', 'technician schedule').\n" - "3. Use `web_search(question)` only to get broader ideas or industry best practices β€” do not use it for final attribute definitions.\n\n" + "3. Use `web_search(question)` only to get broader ideas or industry best practices β€” do not use it for final attribute definitions.\n\n" if settings.TAVILY_API_KEY else "" "When creating the list of attributes:\n" "- Ensure each Dimension or Measure is grounded in the retrieved tables (i.e., it must be derivable from available data).\n" "- Do NOT invent attributes that cannot be linked to existing fields or standard business metrics.\n" diff --git a/src/intugle/core/conceptual_search/agent/tools/web_tools.py b/src/intugle/core/conceptual_search/agent/tools/web_tools.py index f03f227..3eb2e82 100644 --- a/src/intugle/core/conceptual_search/agent/tools/web_tools.py +++ b/src/intugle/core/conceptual_search/agent/tools/web_tools.py @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) -web_search_tool = TavilySearchResults(k=3, tavily_api_key=settings.TAVILY_API_KEY) +web_search_tool = TavilySearchResults(k=3, tavily_api_key=settings.TAVILY_API_KEY) if settings.TAVILY_API_KEY else None @tool @@ -25,6 +25,9 @@ def web_search(question: str) -> str: # Web search log.info(f"Input statement: '{question}'") try: + if not web_search_tool: + log.error("TAVILY_API_KEY is not set. Web search tool is unavailable.") + return Document(page_content="Web search tool is unavailable because TAVILY_API_KEY is not set.") docs = web_search_tool.invoke({"query": question}) web_results = "\n".join([d["content"] for d in docs]) log.info(f"Web search results:\n{web_results}") # Print the search results diff --git a/src/intugle/core/settings.py b/src/intugle/core/settings.py index ac636b5..666b692 100644 --- a/src/intugle/core/settings.py +++ b/src/intugle/core/settings.py @@ -51,8 +51,14 @@ class Settings(BaseSettings): GRAPH_DIR: str = os.path.join(PROJECT_BASE, GRAPH_DIR_NAME) DESCRIPTIONS_DIR: str = os.path.join(PROJECT_BASE, "descriptions") - PROFILES_PATH: str = os.path.join(os.getcwd(), "profiles.yml") + def set_project_base(self, project_base: str): + self.PROJECT_BASE = project_base + self.PROJECT_ID = Project(self.PROJECT_BASE).project_id + self.MODELS_DIR = os.path.join(self.PROJECT_BASE, self.MODELS_DIR_NAME) + self.GRAPH_DIR = os.path.join(self.PROJECT_BASE, self.GRAPH_DIR_NAME) + self.DESCRIPTIONS_DIR = os.path.join(self.PROJECT_BASE, "descriptions") + PROFILES_PATH: str = os.path.join(os.getcwd(), "profiles.yml") PROFILES: dict = load_profiles_configuration(PROFILES_PATH) MCP_SERVER_NAME: str = "intugle" diff --git a/src/intugle/streamlit_app/main.py b/src/intugle/streamlit_app/main.py index 8efccf6..f77202f 100644 --- a/src/intugle/streamlit_app/main.py +++ b/src/intugle/streamlit_app/main.py @@ -159,7 +159,7 @@ def init_io_dirs() -> Dict[str, Path]: paths = { "INPUT_DIR": relpath("input"), "MODIFIED_DIR": relpath("modified_input"), - "ASSET_DIR": relpath("models"), # adjust if your folder name differs + "ASSET_DIR": relpath("intugle/models"), # adjust if your folder name differs # "ICON_DIR": relpath("intugle_assets"), } @@ -169,7 +169,7 @@ def init_io_dirs() -> Dict[str, Path]: # If intugle settings is available, set project base try: # 'settings' should be imported earlier from intugle.core.settings - settings.PROJECT_BASE = str(paths["ASSET_DIR"]) + settings.set_project_base(str(relpath("intugle"))) except NameError: # settings not imported/available; skip without failing the app pass diff --git a/uv.lock b/uv.lock index 593cd29..5d979a9 100644 --- a/uv.lock +++ b/uv.lock @@ -128,6 +128,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "altair" +version = "5.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "narwhals" }, + { name = "packaging" }, + { name = "typing-extensions", marker = "python_full_version < '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/b1/f2969c7bdb8ad8bbdda031687defdce2c19afba2aa2c8e1d2a17f78376d8/altair-5.5.0.tar.gz", hash = "sha256:d960ebe6178c56de3855a68c47b516be38640b73fb3b5111c2a9ca90546dd73d", size = 705305, upload-time = "2024-11-23T23:39:58.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/f3/0b6ced594e51cc95d8c1fc1640d3623770d01e4969d29c0bd09945fafefa/altair-5.5.0-py3-none-any.whl", hash = "sha256:91a310b926508d560fe0148d02a194f38b824122641ef528113d029fcd129f8c", size = 731200, upload-time = "2024-11-23T23:39:56.4Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -259,6 +275,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "azure-core" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/c4/d4ff3bc3ddf155156460bff340bbe9533f99fac54ddea165f35a8619f162/azure_core-1.36.0.tar.gz", hash = "sha256:22e5605e6d0bf1d229726af56d9e92bc37b6e726b141a18be0b4d424131741b7", size = 351139, upload-time = "2025-10-15T00:33:49.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/3c/b90d5afc2e47c4a45f4bba00f9c3193b0417fad5ad3bb07869f9d12832aa/azure_core-1.36.0-py3-none-any.whl", hash = "sha256:fee9923a3a753e94a259563429f3644aaf05c486d45b1215d098115102d91d3b", size = 213302, upload-time = "2025-10-15T00:33:51.058Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography", version = "45.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14' and platform_python_implementation != 'PyPy'" }, + { name = "cryptography", version = "46.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or platform_python_implementation == 'PyPy'" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/8d/1a6c41c28a37eab26dc85ab6c86992c700cd3f4a597d9ed174b0e9c69489/azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456", size = 279826, upload-time = "2025-10-06T20:30:02.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/7b/5652771e24fff12da9dde4c20ecf4682e606b104f26419d139758cc935a6/azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651", size = 191317, upload-time = "2025-10-06T20:30:04.251Z" }, +] + [[package]] name = "backoff" version = "2.2.1" @@ -286,6 +332,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, ] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "boto3" version = "1.40.43" @@ -1338,6 +1393,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, ] +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, +] + [[package]] name = "google-ai-generativelanguage" version = "0.7.0" @@ -1401,6 +1480,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, ] +[[package]] +name = "graphviz" +version = "0.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, +] + [[package]] name = "greenlet" version = "3.2.4" @@ -1415,6 +1503,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/91/ae2eb6b7979e2f9b035a9f612cf70f1bf54aad4e1d125129bef1eae96f19/greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d", size = 584358, upload-time = "2025-08-07T13:18:23.708Z" }, { url = "https://files.pythonhosted.org/packages/f7/85/433de0c9c0252b22b16d413c9407e6cb3b41df7389afc366ca204dbc1393/greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5", size = 1113550, upload-time = "2025-08-07T13:42:37.467Z" }, { url = "https://files.pythonhosted.org/packages/a1/8d/88f3ebd2bc96bf7747093696f4335a0a8a4c5acfcf1b757717c0d2474ba3/greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f", size = 1137126, upload-time = "2025-08-07T13:18:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/f1/29/74242b7d72385e29bcc5563fba67dad94943d7cd03552bac320d597f29b2/greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7", size = 1544904, upload-time = "2025-11-04T12:42:04.763Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e2/1572b8eeab0f77df5f6729d6ab6b141e4a84ee8eb9bc8c1e7918f94eda6d/greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8", size = 1611228, upload-time = "2025-11-04T12:42:08.423Z" }, { url = "https://files.pythonhosted.org/packages/d6/6f/b60b0291d9623c496638c582297ead61f43c4b72eef5e9c926ef4565ec13/greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c", size = 298654, upload-time = "2025-08-07T13:50:00.469Z" }, { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, @@ -1424,6 +1514,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/28a5b2fa42d12b3d7e5614145f0bd89714c34c08be6aabe39c14dd52db34/greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c", size = 1548385, upload-time = "2025-11-04T12:42:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/03f2f0bdd0b0ff9a4f7b99333d57b53a7709c27723ec8123056b084e69cd/greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5", size = 1613329, upload-time = "2025-11-04T12:42:12.928Z" }, { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, @@ -1433,6 +1525,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, @@ -1442,6 +1536,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/1c/53/f9c440463b3057485b8594d7a638bed53ba531165ef0ca0e6c364b5cc807/greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b", size = 1564759, upload-time = "2025-11-04T12:42:19.395Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/3bb4240abdd0a8d23f4f88adec746a3099f0d86bfedb623f063b2e3b4df0/greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929", size = 1634288, upload-time = "2025-11-04T12:42:21.174Z" }, { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, @@ -1449,6 +1545,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, + { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, + { url = "https://files.pythonhosted.org/packages/0d/da/343cd760ab2f92bac1845ca07ee3faea9fe52bee65f7bcb19f16ad7de08b/greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681", size = 1680760, upload-time = "2025-11-04T12:42:25.341Z" }, { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, ] @@ -1689,7 +1787,7 @@ wheels = [ [[package]] name = "intugle" -version = "1.0.10" +version = "1.0.13" source = { editable = "." } dependencies = [ { name = "aiofiles" }, @@ -1730,15 +1828,34 @@ databricks = [ { name = "pyspark" }, { name = "sqlglot" }, ] +postgres = [ + { name = "asyncpg" }, + { name = "sqlglot" }, +] snowflake = [ { name = "snowflake-snowpark-python", extra = ["pandas"] }, + { name = "sqlglot" }, +] +sqlserver = [ + { name = "mssql-python" }, + { name = "sqlglot" }, +] +streamlit = [ + { name = "graphviz" }, + { name = "plotly" }, + { name = "pyngrok" }, + { name = "python-dotenv" }, + { name = "streamlit" }, + { name = "xlsxwriter" }, ] [package.dev-dependencies] dev = [ + { name = "asyncpg" }, { name = "databricks-sql-connector" }, { name = "ipykernel" }, { name = "langfuse" }, + { name = "mssql-python" }, { name = "pysonar" }, { name = "pyspark" }, { name = "pytest" }, @@ -1767,45 +1884,58 @@ requires-dist = [ { name = "aiofiles", specifier = ">=23.2.1" }, { name = "aiohttp", specifier = ">=3.9.5" }, { name = "asyncpg", specifier = ">=0.30.0" }, + { name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.30.0" }, { name = "databricks-sql-connector", marker = "extra == 'databricks'", specifier = ">=4.1.3" }, { name = "duckdb", specifier = ">=1.3.2" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.116.1" }, + { name = "graphviz", marker = "extra == 'streamlit'" }, { name = "langchain", extras = ["anthropic", "google-genai", "openai"], specifier = ">=0.3.27,<1.0.0" }, { name = "langchain-community", specifier = ">=0.3.21,<=0.3.30" }, { name = "langchain-openai", specifier = ">=0.3.28,<=0.3.33" }, { name = "langgraph", specifier = ">=0.6.4" }, { name = "matplotlib", specifier = ">=3.10.5" }, { name = "mcp", extras = ["cli"], specifier = ">=1.12.4" }, + { name = "mssql-python", marker = "extra == 'sqlserver'", specifier = ">=0.13.1" }, { name = "networkx", specifier = ">=3.4.2" }, { name = "nltk", specifier = ">=3.9.1" }, { name = "numpy", specifier = "<=2.3.0" }, { name = "pandas", specifier = ">=2.2.2" }, + { name = "plotly", marker = "extra == 'streamlit'" }, { name = "pyaml", specifier = ">=25.7.0" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "pyfunctional", specifier = ">=1.5.0" }, - { name = "pyspark", marker = "extra == 'databricks'", specifier = ">=3.5.0" }, + { name = "pyngrok", marker = "extra == 'streamlit'", specifier = "==7.4.0" }, + { name = "pyspark", marker = "extra == 'databricks'", specifier = ">=3.5.0,<4.0.0" }, { name = "python-dotenv", specifier = ">=1.1.1" }, + { name = "python-dotenv", marker = "extra == 'streamlit'", specifier = "==1.1.1" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "qdrant-client", specifier = ">=1.15.1" }, { name = "rich", specifier = ">=14.1.0" }, { name = "scikit-learn", specifier = "==1.7.1" }, { name = "snowflake-snowpark-python", extras = ["pandas"], marker = "extra == 'snowflake'", specifier = ">=1.12.0" }, { name = "sqlglot", marker = "extra == 'databricks'", specifier = ">=27.20.0" }, + { name = "sqlglot", marker = "extra == 'postgres'", specifier = ">=27.20.0" }, + { name = "sqlglot", marker = "extra == 'snowflake'", specifier = ">=27.20.0" }, + { name = "sqlglot", marker = "extra == 'sqlserver'", specifier = ">=27.20.0" }, + { name = "streamlit", marker = "extra == 'streamlit'", specifier = "==1.50.0" }, { name = "symspellpy", specifier = ">=6.9.0" }, { name = "tavily-python", specifier = ">=0.1.11" }, { name = "trieregex", specifier = ">=1.0.0" }, { name = "xgboost", specifier = ">=3.0.4" }, + { name = "xlsxwriter", marker = "extra == 'streamlit'", specifier = "==3.2.9" }, ] -provides-extras = ["snowflake", "databricks"] +provides-extras = ["snowflake", "databricks", "postgres", "sqlserver", "streamlit"] [package.metadata.requires-dev] dev = [ + { name = "asyncpg", specifier = ">=0.30.0" }, { name = "databricks-sql-connector", specifier = ">=4.1.3" }, { name = "ipykernel", specifier = ">=6.30.1" }, { name = "langfuse", specifier = "==2.60.4" }, + { name = "mssql-python", specifier = ">=0.13.1" }, { name = "pysonar", specifier = ">=1.2.0.2419" }, - { name = "pyspark", specifier = ">=4.0.1" }, + { name = "pyspark", specifier = ">=3.5.0,<4.0.0" }, { name = "pytest", specifier = ">=8.4.1" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.2.1" }, @@ -2809,6 +2939,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, ] +[[package]] +name = "msal" +version = "1.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography", version = "45.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14' and platform_python_implementation != 'PyPy'" }, + { name = "cryptography", version = "46.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or platform_python_implementation == 'PyPy'" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/c857c46d653e104019a84f22d4494f2119b4fe9f896c92b4b864b3b045cc/msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f", size = 153961, upload-time = "2025-09-22T23:05:48.989Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/dc/18d48843499e278538890dc709e9ee3dea8375f8be8e82682851df1b48b5/msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1", size = 116987, upload-time = "2025-09-22T23:05:47.294Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, +] + +[[package]] +name = "mssql-python" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-identity" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/fa/1d65839b858c975e079bd79f5220be4fa229236693bc38d42cf6c9d27d17/mssql_python-0.13.1-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:9ae88f302b338936ba3d85b6a65af7da27978f7d8870f6aafd86ae1fc64fd318", size = 22590820, upload-time = "2025-10-14T15:06:55.561Z" }, + { url = "https://files.pythonhosted.org/packages/94/75/7bea0ec63145dcf337645b0e5f3a6e2f9925dca6188f4145f3b88c6aad76/mssql_python-0.13.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0277a009191f9d888c66bb7ebd6a370abd5517984d974fc32126b4c43ec4a448", size = 22293324, upload-time = "2025-10-14T15:06:59.023Z" }, + { url = "https://files.pythonhosted.org/packages/ea/5e/2244461025ecec492f8529553c14e833e2d8f35ed92f339ddd15d7206dc5/mssql_python-0.13.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:bab53e15a9072ec432daa5892bb52ae04266455d63b4d08c261837a98e42c73e", size = 22315197, upload-time = "2025-10-14T15:07:01.833Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c3/5ccc8305bc468a7cdef4747bf00f3274aa2dddd78d6ed9b35a8105b9f3ec/mssql_python-0.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed48d45a76ac7eeb4f2e8d77b034d46a3dea6bbcebadba41d1725f0845dbc989", size = 22233248, upload-time = "2025-10-14T15:07:04.424Z" }, + { url = "https://files.pythonhosted.org/packages/1a/61/d57da2f7c173beaefec29ef52a6ff202b14916f14579872359d72d542e47/mssql_python-0.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a1ff57dd3be57f357ef8bc3bdbc8835029a4a79cc2032b493964c60afd888989", size = 22239361, upload-time = "2025-10-14T15:07:06.789Z" }, + { url = "https://files.pythonhosted.org/packages/25/06/e2bfade798c98a2214ec58356ac8dd94d3dc312cc97d008cbc2ffec7b859/mssql_python-0.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:732c9bf5d54ed011fa2913f4113df023fe871f3c429deaeb342775df69e36519", size = 12473080, upload-time = "2025-10-14T15:07:09.133Z" }, + { url = "https://files.pythonhosted.org/packages/55/42/0bb5a7e737949fd2119af852d210202e973974e2f731ec2d29f88963b3ce/mssql_python-0.13.1-cp311-cp311-macosx_15_0_universal2.whl", hash = "sha256:f2169228457f9bbfce156bc9a1b17eb6e38311431186cbbbe262e89ae7294a4e", size = 22591169, upload-time = "2025-10-14T15:07:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/b0/59/62b280c0785c324bfe7bf9e4f36e7852d0506de0dc7f8da98d22ca827cfa/mssql_python-0.13.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:8345887cac017babb54998e517c8693f2d4399c8adb25872dd6f07e8ccce3aba", size = 23156237, upload-time = "2025-10-14T15:07:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e3/5356d6e623803d0dfc280e2fa2029ceb56d8b16692d12cbf3e1ceb8d5f0f/mssql_python-0.13.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:3568eff3bbcc2a4a514f355683454cc21041e44af8603b4bced2193f25826b51", size = 23199704, upload-time = "2025-10-14T15:07:16.759Z" }, + { url = "https://files.pythonhosted.org/packages/51/cf/a08627e814144a2480af6e8e83356a40b1edfda1531a3681d6ef77fa87bf/mssql_python-0.13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b70e0e73de5ba9e6a34b540adc4ab4acca01211c931acd8513f0d6868736d7fb", size = 23036583, upload-time = "2025-10-14T15:07:19.387Z" }, + { url = "https://files.pythonhosted.org/packages/65/63/4cfaa1804554aae264597457d35dd6f86f74a9c2487700c26d8f36ca90ff/mssql_python-0.13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f021abdc9e04ff661d75f115eb136795b27769b032eba389d1482167bcf1e9d7", size = 23048855, upload-time = "2025-10-14T15:07:22.241Z" }, + { url = "https://files.pythonhosted.org/packages/41/4f/6d715716567b804712efa8835b672359bf7fe71064ac34728f645b9dbcf6/mssql_python-0.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:536e91061b22ac63013c553bef80fd160796ad714ae6909675f0d976bacf9821", size = 12474099, upload-time = "2025-10-14T15:07:25.027Z" }, + { url = "https://files.pythonhosted.org/packages/9c/45/46e36fa1a478c8e573b7a1cdc062ad041a42bb66b070e4cdc94dbe6e2440/mssql_python-0.13.1-cp311-cp311-win_arm64.whl", hash = "sha256:cc16cb0ac113d9a83ae3207217de5f6062669e1329cab536181d21aa30adef08", size = 15616130, upload-time = "2025-10-14T15:07:28.269Z" }, + { url = "https://files.pythonhosted.org/packages/79/5d/d157d5f88047051b266c7c6ecbb02c3086650c0c2aa63cbda7f037bae652/mssql_python-0.13.1-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:4875049e6c29c48dcc2ce42ad83d04f0c25888ad62668c470a9d5ba1aede80fe", size = 22595707, upload-time = "2025-10-14T15:07:31.018Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/c07f8dbe37ff583e0d8f9f9e554c6562da8184261f3cf179d85b09b6d03d/mssql_python-0.13.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:412e870db92674d1d24d8afabdae8640e9cff39c673045a77e5e32e34d582f15", size = 24021144, upload-time = "2025-10-14T15:07:33.949Z" }, + { url = "https://files.pythonhosted.org/packages/64/eb/34ad320fff5f6e128121015298cb4d80c37d95591f54f64455aab68d4297/mssql_python-0.13.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4964a13172fa1ccff4d1d6d2c19b3f1a810bf12b28f2d16c6e8a9cd862f776ba", size = 24085959, upload-time = "2025-10-14T15:07:36.859Z" }, + { url = "https://files.pythonhosted.org/packages/16/11/d1e76b16648b666ccb7a221fcf4fb84422b2fd2185b19ad07173e60d52c8/mssql_python-0.13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e96e2c2c062f3f4edf482f42361863f8341c02855bdc0d058d8a016c05060f75", size = 23842405, upload-time = "2025-10-14T15:07:39.554Z" }, + { url = "https://files.pythonhosted.org/packages/71/0d/b6e26dcec9bcdab1d0cdc05ea521728708efe01e6fe2c049f1c1eaef65e8/mssql_python-0.13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db2f7b965d22d316ab373b9fc82987333d088f55040549beba19fa2d883e86c6", size = 23860413, upload-time = "2025-10-14T15:07:41.981Z" }, + { url = "https://files.pythonhosted.org/packages/63/fa/dc48caedb29d6a07513f26d0c2b487fabc3a08a779da2a2f84a5a71bd589/mssql_python-0.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:07372d846c89a12a8287fc469a6a80a04ef322bdcdb35b5d3f4780b4d801a353", size = 12476596, upload-time = "2025-10-14T15:07:44.742Z" }, + { url = "https://files.pythonhosted.org/packages/da/7b/bca1199cfb5e8b161482a9581ab56509db800397db030f964bb252288193/mssql_python-0.13.1-cp312-cp312-win_arm64.whl", hash = "sha256:d65afa8ea8c7462d24a29c6fe595497c8bcae7cb9d51a56b4e721f71df17869b", size = 15618258, upload-time = "2025-10-14T15:07:47.131Z" }, + { url = "https://files.pythonhosted.org/packages/fe/13/73ea47eae7fd3965c4e4e3050d6bdac34d16c698e3f7cbf4d67cc870471c/mssql_python-0.13.1-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:8c07bac13ef148f4b832d66e562e379746df6a6e1b34f9e52e1a2945a48f77a1", size = 22596048, upload-time = "2025-10-14T15:07:50.15Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/e6fa633ee35d5f51c7f6f0d52bba1e1fd24521cfda2c2e58c037b7c9567b/mssql_python-0.13.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:56aac87b3a711a0be943118d87bfb1517466bf425eb67b751095c4e84ecfd5c9", size = 24886235, upload-time = "2025-10-14T15:07:54.211Z" }, + { url = "https://files.pythonhosted.org/packages/ef/35/4048438da194dcf7662baab0ddfa317f444b044ebd35dc4e82b2bbb50113/mssql_python-0.13.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:005d6775bdbcc8fdac91d4baf8494cab4426db053b89a3bf5046e6f94999cd7a", size = 24972422, upload-time = "2025-10-14T15:07:57.182Z" }, + { url = "https://files.pythonhosted.org/packages/09/9b/e9a2d43522f55715ab1e359b7f190984433f1acda457e53526d3ce64342c/mssql_python-0.13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0fb7367c6b64629015d87fef60153fa92a5a46c4eb2893510b919ae716b03db3", size = 24648630, upload-time = "2025-10-14T15:07:59.73Z" }, + { url = "https://files.pythonhosted.org/packages/60/aa/b6b063cb7e7d02a2c3887da4376910fa7d5ccc24b6608b1dd8b5489632ad/mssql_python-0.13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:82c2ac027df2ff38dd3962c730b642d1b1371976006451a1a19328952e8744f0", size = 24672185, upload-time = "2025-10-14T15:08:02.548Z" }, + { url = "https://files.pythonhosted.org/packages/0e/d8/0ccdbfe9adfc60180d2f2f8ca63ab4e701f9a4a89ea54668ce493cba8634/mssql_python-0.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:e98ae2e0ede7cd79a5ad360cc0480fbfc9f99ead5b9b545458c7c164851e7f5e", size = 12476560, upload-time = "2025-10-14T15:08:05.182Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ab/358edc6fdf50e4861e8e432c54f9d932046348f5deffb2d6b571647fc9ca/mssql_python-0.13.1-cp313-cp313-win_arm64.whl", hash = "sha256:bf99c01d5d587a756263c54a2fca6124f099875fff29145863cafea75adaa476", size = 15618238, upload-time = "2025-10-14T15:08:07.721Z" }, +] + [[package]] name = "multidict" version = "6.6.4" @@ -2920,6 +3114,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "narwhals" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/a2/25208347aa4c2d82a265cf4bc0873aaf5069f525c0438146821e7fc19ef5/narwhals-2.11.0.tar.gz", hash = "sha256:d23f3ea7efc6b4d0355444a72de6b8fa3011175585246c3400c894a7583964af", size = 589233, upload-time = "2025-11-10T16:28:35.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/a1/4d21933898e23b011ae0528151b57a9230a62960d0919bf2ee48c7f5c20a/narwhals-2.11.0-py3-none-any.whl", hash = "sha256:a9795e1e44aa94e5ba6406ef1c5ee4c172414ced4f1aea4a79e5894f0c7378d4", size = 423069, upload-time = "2025-11-10T16:28:33.522Z" }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -3493,6 +3696,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, ] +[[package]] +name = "plotly" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/e6/b768650072837505804bed4790c5449ba348a3b720e27ca7605414e998cd/plotly-6.4.0.tar.gz", hash = "sha256:68c6db2ed2180289ef978f087841148b7efda687552276da15a6e9b92107052a", size = 7012379, upload-time = "2025-11-04T17:59:26.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/ae/89b45ccccfeebc464c9233de5675990f75241b8ee4cd63227800fdf577d1/plotly-6.4.0-py3-none-any.whl", hash = "sha256:a1062eafbdc657976c2eedd276c90e184ccd6c21282a5e9ee8f20efca9c9a4c5", size = 9892458, upload-time = "2025-11-04T17:59:22.622Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -3677,11 +3893,11 @@ wheels = [ [[package]] name = "py4j" -version = "0.10.9.9" +version = "0.10.9.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/db/ea0203e495be491c85af87b66e37acfd3bf756fd985f87e46fc5e3bf022c/py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533", size = 203008, upload-time = "2025-01-15T03:53:15.648Z" }, + { url = "https://files.pythonhosted.org/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b", size = 200481, upload-time = "2022-08-12T22:49:07.05Z" }, ] [[package]] @@ -3890,6 +4106,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/d6/887a1ff844e64aa823fb4905978d882a633cfe295c32eacad582b78a7d8b/pydantic_settings-2.11.0-py3-none-any.whl", hash = "sha256:fe2cea3413b9530d10f3a5875adffb17ada5c1e1bab0b2885546d7310415207c", size = 48608, upload-time = "2025-09-24T14:19:10.015Z" }, ] +[[package]] +name = "pydeck" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/40e14e196864a0f61a92abb14d09b3d3da98f94ccb03b49cf51688140dab/pydeck-0.9.1.tar.gz", hash = "sha256:f74475ae637951d63f2ee58326757f8d4f9cd9f2a457cf42950715003e2cb605", size = 3832240, upload-time = "2024-05-10T15:36:21.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/4c/b888e6cf58bd9db9c93f40d1c6be8283ff49d88919231afe93a6bcf61626/pydeck-0.9.1-py2.py3-none-any.whl", hash = "sha256:b3f75ba0d273fc917094fa61224f3f6076ca8752b93d46faf3bcfd9f9d59b038", size = 6900403, upload-time = "2024-05-10T15:36:17.36Z" }, +] + [[package]] name = "pyfakefs" version = "5.9.3" @@ -3930,6 +4160,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, ] +[package.optional-dependencies] +crypto = [ + { name = "cryptography", version = "45.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14' and platform_python_implementation != 'PyPy'" }, + { name = "cryptography", version = "46.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or platform_python_implementation == 'PyPy'" }, +] + +[[package]] +name = "pyngrok" +version = "7.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/40/f06cbabac68bf2ff33b499d32373dedc20ebec3da814c26f82281f3c9bec/pyngrok-7.4.0.tar.gz", hash = "sha256:1f786fed8d156b99bffd713096a7ec8c81181f949c8f5c920f6820e92d6b9876", size = 44530, upload-time = "2025-09-23T14:47:42.642Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/60/00c4570b6de596a87bd05cb828eaf507e6b9ce171ad797e90fe5e8ecbd6e/pyngrok-7.4.0-py3-none-any.whl", hash = "sha256:719ee7bda031c7c3c80e7722aa3103961ac58bfae2167881d57fef28f5c8917a", size = 25434, upload-time = "2025-09-23T14:47:41.229Z" }, +] + [[package]] name = "pyopenssl" version = "25.3.0" @@ -3971,12 +4219,12 @@ wheels = [ [[package]] name = "pyspark" -version = "4.0.1" +version = "3.5.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "py4j" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/40/1414582f16c1d7b051c668c2e19c62d21a18bd181d944cb24f5ddbb2423f/pyspark-4.0.1.tar.gz", hash = "sha256:9d1f22d994f60369228397e3479003ffe2dd736ba79165003246ff7bd48e2c73", size = 434204896, upload-time = "2025-09-06T07:15:57.091Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/65/2764b1840aa6ea5f7e668a36702c3128ac54d0b861d3d57669b2453469a6/pyspark-3.5.7.tar.gz", hash = "sha256:80e36514e0c5c126d35a26adf4f405cd4a69de96a8ea8a3c1e9a65fce2fb4eaa", size = 317370347, upload-time = "2025-09-23T19:46:10.326Z" } [[package]] name = "pytest" @@ -4986,6 +5234,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -5177,6 +5434,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736, upload-time = "2025-09-13T08:41:03.869Z" }, ] +[[package]] +name = "streamlit" +version = "1.50.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altair" }, + { name = "blinker" }, + { name = "cachetools" }, + { name = "click" }, + { name = "gitpython" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "pyarrow" }, + { name = "pydeck" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "tornado" }, + { name = "typing-extensions" }, + { name = "watchdog", marker = "sys_platform != 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/f6/f7d3a0146577c1918439d3163707040f7111a7d2e7e2c73fa7adeb169c06/streamlit-1.50.0.tar.gz", hash = "sha256:87221d568aac585274a05ef18a378b03df332b93e08103fffcf3cd84d852af46", size = 9664808, upload-time = "2025-09-23T19:24:00.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/38/991bbf9fa3ed3d9c8e69265fc449bdaade8131c7f0f750dbd388c3c477dc/streamlit-1.50.0-py3-none-any.whl", hash = "sha256:9403b8f94c0a89f80cf679c2fcc803d9a6951e0fba542e7611995de3f67b4bb3", size = 10068477, upload-time = "2025-09-23T19:23:57.245Z" }, +] + [[package]] name = "symspellpy" version = "6.9.0" @@ -5275,6 +5562,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/79/bcf350609f3a10f09fe4fc207f132085e497fdd3612f3925ab24d86a0ca0/tiktoken-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:2177ffda31dec4023356a441793fed82f7af5291120751dee4d696414f54db0c", size = 883901, upload-time = "2025-08-08T23:57:59.359Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomli" version = "2.2.1" @@ -5525,6 +5821,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018, upload-time = "2024-10-14T23:38:10.888Z" }, ] +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + [[package]] name = "watchfiles" version = "1.1.0" @@ -5793,6 +6107,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/5a/f43bad68b31269a72bdd66102732ea4473e98f421ee9f71379e35dcb56f5/xgboost-3.0.5-py3-none-win_amd64.whl", hash = "sha256:660774249d28a729ba8d22dd3d2c048c56e58f65a683b25ef3252e3383fe956f", size = 56826727, upload-time = "2025-09-05T09:23:55.462Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "xxhash" version = "3.5.0" From 7799257bf2d28d8c01946de1339a1a042ce6eec9 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 11 Nov 2025 14:19:09 +0530 Subject: [PATCH 24/26] conditional fix in prompt --- .../core/conceptual_search/agent/prompts.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/intugle/core/conceptual_search/agent/prompts.py b/src/intugle/core/conceptual_search/agent/prompts.py index 1198c4d..7ceb904 100644 --- a/src/intugle/core/conceptual_search/agent/prompts.py +++ b/src/intugle/core/conceptual_search/agent/prompts.py @@ -6,21 +6,27 @@ [ ( "system", - "You are a data product assistant. Your task is to build a high-quality list of Dimensions and Measures for a business data product.\n\n" - "Follow these steps:\n" - "1. Use `retrieve_existing_data_products(statement)` to identify similar data products and gather candidate Dimensions and Measures.\n" - "2. Use `retrieve_table_details(statement)` to get relevant database tables. If the initial query does not return all required information, do not hesitate to run multiple queries with refined or different search terms (e.g., 'customer data', 'order transactions', 'technician schedule').\n" - "3. Use `web_search(question)` only to get broader ideas or industry best practices β€” do not use it for final attribute definitions.\n\n" if settings.TAVILY_API_KEY else "" - "When creating the list of attributes:\n" - "- Ensure each Dimension or Measure is grounded in the retrieved tables (i.e., it must be derivable from available data).\n" - "- Do NOT invent attributes that cannot be linked to existing fields or standard business metrics.\n" - "- Avoid generalities β€” prefer specific, field-aligned attributes over vague concepts.\n" - "- Use precise, unambiguous names.\n" - "- Eliminate duplicates and redundancy (e.g., avoid 'Total Revenue' and 'Revenue Total').\n" - "- Group Measures and Dimensions logically if possible.\n" - "- Include a short description for each.\n" - "- Tag each as either a 'Dimension' or a 'Measure'.\n\n" - "Once ready, persist the final list using `save_data_product(attribute_data)` in the following format:\n", + ( + "You are a data product assistant. Your task is to build a high-quality list of Dimensions and Measures for a business data product.\n\n" + "Follow these steps:\n" + "1. Use `retrieve_existing_data_products(statement)` to identify similar data products and gather candidate Dimensions and Measures.\n" + "2. Use `retrieve_table_details(statement)` to get relevant database tables. If the initial query does not return all required information, do not hesitate to run multiple queries with refined or different search terms (e.g., 'customer data', 'order transactions', 'technician schedule').\n" + "3. Use `web_search(question)` only to get broader ideas or industry best practices β€” do not use it for final attribute definitions.\n\n" + if settings.TAVILY_API_KEY + else "" + ) + + ( + "When creating the list of attributes:\n" + "- Ensure each Dimension or Measure is grounded in the retrieved tables (i.e., it must be derivable from available data).\n" + "- Do NOT invent attributes that cannot be linked to existing fields or standard business metrics.\n" + "- Avoid generalities β€” prefer specific, field-aligned attributes over vague concepts.\n" + "- Use precise, unambiguous names.\n" + "- Eliminate duplicates and redundancy (e.g., avoid 'Total Revenue' and 'Revenue Total').\n" + "- Group Measures and Dimensions logically if possible.\n" + "- Include a short description for each.\n" + "- Tag each as either a 'Dimension' or a 'Measure'.\n\n" + "Once ready, persist the final list using `save_data_product(attribute_data)` in the following format:\n" + ), ), ("human", "{messages}"), ] From 29e25ceeb00bd72989c66c467420b5a855a4b8b7 Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 11 Nov 2025 16:40:21 +0530 Subject: [PATCH 25/26] added docs and notebook --- README.md | 4 +- conceptual_search_quickstart.ipynb | 158 ---------- .../data-product/conceptual-search.md | 119 +++++++ .../docs/core-concepts/data-product/index.md | 1 + docsite/docs/examples.md | 1 + notebooks/quickstart_conceptual_search.ipynb | 297 ++++++++++++++++++ pyproject.toml | 4 +- test_col_search.py | 24 -- test_table_search.py | 25 -- uv.lock | 4 +- 10 files changed, 425 insertions(+), 212 deletions(-) delete mode 100644 conceptual_search_quickstart.ipynb create mode 100644 docsite/docs/core-concepts/data-product/conceptual-search.md create mode 100644 notebooks/quickstart_conceptual_search.ipynb delete mode 100644 test_col_search.py delete mode 100644 test_table_search.py diff --git a/README.md b/README.md index 0e41de0..bf35b95 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ Intugle’s GenAI-powered open-source Python library builds a semantic data mode * **Semantic Data Model -** Transform raw, fragmented datasets into an intelligent semantic graph that captures entities, relationships, and context β€” the foundation for connected intelligence. * **Business Glossary & Semantic Search:** Auto-generate a business glossary and enable search that understands meaning, not just keywords β€” making data more accessible across technical and business users. * **Data Products -** Instantly generate SQL and reusable data products enriched with context, eliminating manual pipelines and accelerating data-to-insight. +* **Conceptual Search -** Generate data product plans from natural language queries, bridging the gap between business questions and executable data product definitions. Learn more in the [documentation](https://intugle.github.io/data-tools/docs/core-concepts/data-product/conceptual-search). + ## Getting Started @@ -107,7 +109,7 @@ For a detailed, hands-on introduction to the project, please see our quickstart | **Native Snowflake with Cortex Analyst [ Tech Manufacturing ]** | [`quickstart_native_snowflake.ipynb`](notebooks/quickstart_native_snowflake.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_native_snowflake.ipynb) | | **Native Databricks with AI/BI Genie [ Tech Manufacturing ]** | [`quickstart_native_databricks.ipynb`](notebooks/quickstart_native_databricks.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_native_databricks.ipynb) | | **Streamlit App** | [`quickstart_streamlit.ipynb`](notebooks/quickstart_streamlit.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_streamlit.ipynb) | - +| **Conceptual Search** | [`quickstart_conceptual_search.ipynb`](notebooks/quickstart_conceptual_search.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_conceptual_search.ipynb) | These datasets will take you through the following steps: * **Generate Semantic Model** β†’ The unified layer that transforms fragmented datasets, creating the foundation for connected intelligence. diff --git a/conceptual_search_quickstart.ipynb b/conceptual_search_quickstart.ipynb deleted file mode 100644 index 5f3dc86..0000000 --- a/conceptual_search_quickstart.ipynb +++ /dev/null @@ -1,158 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Conceptual Search Quickstart\n", - "\n", - "This notebook demonstrates how to use the new **Conceptual Search** functionality in the `intugle` library. The process involves two main steps:\n", - "\n", - "1. **Generate a Data Product Plan:** You provide a high-level goal in natural language (e.g., \"I want a report on patient allergies\"), and the Planner Agent generates a structured CSV file (`attributes.csv`) detailing the required dimensions and measures.\n", - "2. **Build the Data Product:** The Builder Agent takes the generated plan and maps each attribute to the most relevant columns in your data sources, creating a final mapping file (`column_logic_results.csv`).\n", - "\n", - "**Prerequisite:** This workflow requires a built `SemanticModel`, as the agents rely on the manifest YAML files it generates." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 1: Build the Semantic Model\n", - "\n", - "First, we need to build a `SemanticModel` from our data sources. This will profile the data, discover relationships, and generate the manifest files that the Conceptual Search agents need to understand your data landscape." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from dotenv import load_dotenv\n", - "load_dotenv(override=True)\n", - "\n", - "from intugle import SemanticModel\n", - "\n", - "# Define the paths to your datasets\n", - "# For this example, we'll use the sample healthcare data included in the repo.\n", - "datasets = {\n", - " \"patients\": {\"path\": \"../sample_data/healthcare/patients.csv\", \"type\": \"csv\"},\n", - " \"allergies\": {\"path\": \"../sample_data/healthcare/allergies.csv\", \"type\": \"csv\"},\n", - " \"encounters\": {\"path\": \"../sample_data/healthcare/encounters.csv\", \"type\": \"csv\"},\n", - " \"conditions\": {\"path\": \"../sample_data/healthcare/conditions.csv\", \"type\": \"csv\"}\n", - "}\n", - "\n", - "# Build the semantic model\n", - "# This will generate the necessary YAML files in your project root directory.\n", - "sm = SemanticModel(datasets, domain=\"Healthcare\")\n", - "sm.build()\n", - "\n", - "print(\"Semantic Model has been successfully built.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 2: Generate a Data Product Plan\n", - "\n", - "Now that the semantic model is built, we can use the `ConceptualSearch` class to generate a plan for our desired data product. We'll define our goal in a simple, natural language query." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from intugle import DataProduct\n", - "\n", - "data_product = DataProduct()\n", - "\n", - "\n", - "query = \"patient 360\"\n", - "\n", - "\n", - "data_product_plan = await data_product.plan(query, \"patient 360 view\")\n", - "\n", - "# Display the plan. This is a special object that can be modified in code.\n", - "data_product_plan\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# data_product_plan.disable_attribute(\"Date of Birth\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 3: Build the Data Product (Map Plan to Columns)\n", - "\n", - "With the plan generated, we can now pass it to the Builder Agent. This agent will intelligently search the semantic model to find the best table and column for each attribute in the plan." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import pandas as pd\n", - "# from intugle.core.conceptual_search.search import ConceptualSearch\n", - "# conceptual_search = ConceptualSearch(force_recreate=False)\n", - "\n", - "\n", - "# plan_df = pd.read_csv(\"attributes.csv\")\n", - "\n", - "# The 'plan_df' is the DataFrame we generated in the previous step\n", - "if data_product_plan is not None:\n", - " etl_model = await data_product.create_etl_model_from_plan(data_product_plan)\n", - "\n", - "print(etl_model.model_dump_json(indent=2))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result_dataset = data_product.build(etl_model)\n", - "\n", - "print(\"-- Generated SQL Query --\")\n", - "print(result_dataset.sql_query)\n", - "\n", - "print(\"\\n-- Resulting Data (First 5 Rows) --\")\n", - "result_dataset.to_df().head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "intugle", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docsite/docs/core-concepts/data-product/conceptual-search.md b/docsite/docs/core-concepts/data-product/conceptual-search.md new file mode 100644 index 0000000..af1f343 --- /dev/null +++ b/docsite/docs/core-concepts/data-product/conceptual-search.md @@ -0,0 +1,119 @@ +--- +sidebar_position: 8 +title: Conceptual Search +--- + +# Conceptual Search + +:::info Experimental Feature +Conceptual Search is an experimental feature. The API and functionality may change in future releases. +::: + +Conceptual Search is an AI-powered feature that allows you to generate a data product plan from a natural language query. It bridges the gap between a high-level business question and a concrete, executable data product definition. + +## Overview + +At its core, Conceptual Search uses a sophisticated, two-stage process orchestrated by AI agents and knowledge graphs: + +1. **Knowledge Graphs**: The system builds knowledge graphs for both database tables and columns. Nodes represent tables/columns, and edges connect conceptually related items based on semantic similarity and shared concepts extracted by an LLM. +2. **Graph-Based Retrievers**: When you search, the system uses a hybrid approach of vector search and graph traversal to find relevant tables and columns, even if they are not direct keyword matches. +3. **AI Agents**: The process is managed by two LangChain/LangGraph agents: a `DataProductPlannerAgent` and a `DataProductBuilderAgent`. + +## The Two-Stage Workflow + +### Stage 1: Planning + +The goal of this stage is to convert a vague user request (e.g., "customer churn metrics") into a structured `DataProductPlan`, which is a well-defined list of dimensions and measures. + +1. **Input**: A natural language query. +2. **Agent's Task**: The `DataProductPlannerAgent` uses its tools to find relevant database tables and existing data products. +3. **Output**: The agent produces a `DataProductPlan` object, which can be reviewed and modified by the user. + +:::tip User Validation is Key +The `DataProductPlan` generated by the AI is a starting point. It is crucial to review and validate this plan to ensure it aligns with your business requirements before proceeding to the building stage. +::: + +### Stage 2: Building + +This stage takes the abstract `DataProductPlan` and maps each attribute to a specific, physical database column, defining its logic (e.g., aggregation for measures). + +1. **Input**: The `DataProductPlan` from Stage 1. +2. **Agent's Task**: The `DataProductBuilderAgent` iterates through each attribute in the plan, using the graph-based column retriever to find the most relevant physical column. +3. **Output**: The collected mappings are assembled into a final `ETLModel`, which is a complete, machine-readable definition of the data product, ready to be used to generate a SQL query. + +## Usage Example + +```python +from intugle import DataProduct + +dp = DataProduct() + +# 1. Generate a plan from a natural language query +plan = await dp.plan(query="top 10 customers by their total purchase amount") + +# 2. Review and modify the plan +print("Original Plan:") +plan.display() + +plan.rename_attribute("total purchase amount", "Total Spend") +plan.disable_attribute("customer address") # Assuming this was in the plan + +print("\nModified Plan:") +plan.display() + + +# 3. Create the ETL model from the modified plan +etl_model = await dp.create_etl_model_from_plan(plan) + +# 4. Build the data product +result_dataset = dp.build(etl=etl_model) + +# 5. Access the results +print(result_dataset.to_df()) +``` + + +## Modifying the Data Product Plan + +The `DataProductPlan` object is not just a static output; it's an interactive object that you can modify to refine the AI's suggestions. This allows you to correct any misunderstandings or add your own domain knowledge to the plan. + +Here are the available methods to modify the plan: + +| Method | Description | Example | +| ---------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `rename_attribute(old, new)` | Renames an existing attribute. | `plan.rename_attribute('Customer ID', 'Client Identifier')` | +| `set_attribute_description(name, desc)` | Updates the description of an attribute. | `plan.set_attribute_description('Client Identifier', 'The unique ID for each client')` | +| `set_attribute_classification(name, class)` | Changes the classification to 'Dimension' or 'Measure'. | `plan.set_attribute_classification('Total Sales', 'Measure')` | +| `disable_attribute(name)` | Deactivates an attribute so it won't be included in the final data product. | `plan.disable_attribute('Customer Address')` | +| `enable_attribute(name)` | Reactivates a previously disabled attribute. | `plan.enable_attribute('Customer Address')` | +| `to_df()` | Returns the final plan as a pandas DataFrame with only active attributes. | `final_plan_df = plan.to_df()` | + +## Qdrant Server Requirement + +Conceptual Search utilizes [Qdrant](https://qdrant.tech/) as its vector database for efficient retrieval of relevant tables and columns. Therefore, a running Qdrant instance is required. + +You can easily set up a Qdrant server using Docker: + +```bash +docker run -d -p 6333:6333 -p 6334:6334 \ + -v qdrant_storage:/qdrant/storage:z \ + --name qdrant qdrant/qdrant +``` + +After starting the Qdrant server, you need to configure its URL and API key (if authorization is used) in your environment variables: + +```bash +export QDRANT_URL="http://localhost:6333" +export QDRANT_API_KEY="your-qdrant-api-key" # if authorization is used +``` + +## Enhancing Performance with Tavily Web Search + +For better performance and more contextually aware data product plans, it is recommended to use the Tavily web search tool. This allows the planning agent to research industry-best practices and common metrics related to your query. + +To enable this feature, you need to get a API key from [Tavily](https://tavily.com/) and set it as an environment variable: + +```bash +export TAVILY_API_KEY="your-tavily-api-key" +``` + diff --git a/docsite/docs/core-concepts/data-product/index.md b/docsite/docs/core-concepts/data-product/index.md index 868f8f4..eca6c67 100644 --- a/docsite/docs/core-concepts/data-product/index.md +++ b/docsite/docs/core-concepts/data-product/index.md @@ -93,3 +93,4 @@ The `DataProduct` class provides a powerful way to query your connected data wit * **[Aggregations](./aggregations.md)**: Understand how to perform grouping and aggregation functions like `COUNT` and `SUM`. * **[Joins](./joins.md)**: Discover how the builder automatically handles joins between tables. * **[Advanced Examples](./advanced-examples.md)**: See how to combine these concepts to build complex data products. +* **[Conceptual Search](./conceptual-search.md)**: Learn how to generate data products from natural language queries. diff --git a/docsite/docs/examples.md b/docsite/docs/examples.md index af1daf9..15bf34c 100644 --- a/docsite/docs/examples.md +++ b/docsite/docs/examples.md @@ -13,6 +13,7 @@ For a detailed, hands-on introduction to the project, please see our quickstart | **Tech Manufacturing** | [`quickstart_tech_manufacturing.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_tech_manufacturing.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_tech_manufacturing.ipynb) | | **FMCG** | [`quickstart_fmcg.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_fmcg.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_fmcg.ipynb) | | **Sports Media** | [`quickstart_sports_media.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_sports_media.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_sports_media.ipynb) | +| **Conceptual Search** | [`quickstart_conceptual_search.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_conceptual_search.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_conceptual_search.ipynb) | | **Databricks Unity Catalog [Health Care]** | [`quickstart_healthcare_databricks.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_healthcare_databricks.ipynb) | Databricks Notebook Only | | **Snowflake Horizon Catalog [ FMCG ]** | [`quickstart_fmcg_snowflake.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_fmcg_snowflake.ipynb) | Snowflake Notebook Only | | **Native Snowflake with Cortex Analyst [ Tech Manufacturing ]** | [`quickstart_native_snowflake.ipynb`](https://github.com/Intugle/data-tools/blob/main/notebooks/quickstart_native_snowflake.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Intugle/data-tools/blob/main/notebooks/quickstart_native_snowflake.ipynb) | diff --git a/notebooks/quickstart_conceptual_search.ipynb b/notebooks/quickstart_conceptual_search.ipynb new file mode 100644 index 0000000..cc59374 --- /dev/null +++ b/notebooks/quickstart_conceptual_search.ipynb @@ -0,0 +1,297 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Quickstart: Conceptual Search\n", + "\n", + "This notebook provides a hands-on walkthrough of the **Conceptual Search** feature in the `intugle` library. You'll learn how to use natural language to generate, refine, and build a unified data product from a semantic model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "First, let's install the `intugle` library." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install intugle -q" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Environment Variables\n", + "\n", + "Conceptual Search relies on Large Language Models (LLMs) for its AI capabilities. You'll need to configure an LLM provider and API key. For this example, we'll use OpenAI.\n", + "\n", + "## 1. LLM Configuration\n", + "\n", + "Before running the project, you need to configure a Large Language Model (LLM). This is used for tasks like generating business glossaries and predicting links between tables. You will also need to set up Qdrant and provide an OpenAI API key. For detailed setup instructions, please refer to the [README.md](README.md) file.\n", + "\n", + "You can configure the necessary services by setting the following environment variables:\n", + "\n", + "* `LLM_PROVIDER`: The LLM provider and model to use (e.g., `openai:gpt-3.5-turbo`). The format follows langchain's format for initializing chat models. Checkout how to specify your model [here](https://python.langchain.com/docs/integrations/chat/)\n", + "* `API_KEY`: Your API key for the LLM provider. The exact name of the variable may vary from provider to provider (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).\n", + "* `QDRANT_URL`: The URL of your Qdrant instance (e.g., `http://localhost:6333`).\n", + "* `QDRANT_API_KEY`: Your API key for the Qdrant instance, if authorization is enabled.\n", + "* `EMBEDDING_MODEL_NAME`: The embedding model to use. The format follows LangChain's conventions for initializing embedding models (e.g., `openai:ada`, `azure_openai:ada`).\n", + "* `OPENAI_API_KEY`: Your OpenAI API key, required if you are using an OpenAI embedding model.\n", + "* `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `OPENAI_API_VERSION`: Your Azure OpenAI credentials, required if you are using an Azure OpenAI embedding model.\n", + "\n", + "For the best results, we also recommend setting a **Tavily API key**, which allows the planning agent to perform web searches for better contextual understanding.\n", + "\n", + "Here's an example of how to set these variables in your environment:\n", + "\n", + "```bash\n", + "export LLM_PROVIDER=\"openai:gpt-3.5-turbo\"\n", + "export OPENAI_API_KEY=\"your-openai-api-key\"\n", + "```\n", + "Alternatively, you can set them in the notebook like this:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"LLM_PROVIDER\"] = \"openai:gpt-3.5-turbo\"\n", + "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-api-key\" # Replace with your actual key\n", + "\n", + "# Qdrant Configuration\n", + "os.environ[\"QDRANT_URL\"] = \"http://localhost:6333\"\n", + "os.environ[\"QDRANT_API_KEY\"] = \"\" # if authorization is used\n", + "os.environ[\"EMBEDDING_MODEL_NAME\"] = \"openai:ada\"\n", + "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-api-key\"\n", + "\n", + "# For Azure OpenAI models\n", + "os.environ[\"EMBEDDING_MODEL_NAME\"] = \"azure_openai:ada\"\n", + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"your-azure-openai-api-key\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"your-azure-openai-endpoint\"\n", + "os.environ[\"OPENAI_API_VERSION\"] = \"your-openai-api-version\"\n", + "\n", + "# TAVILY\n", + "os.environ[\"TAVILY_API_KEY\"] = \"YOUR_TAVILY_API_KEY\" # Optional, but recommended" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Download Sample Data\n", + "\n", + "We'll use the healthcare dataset for this demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import requests\n", + "\n", + "raw_datasets='healthcare'\n", + "api_url = f\"https://api.github.com/repos/Intugle/data-tools/contents/sample_data/{raw_datasets}\"\n", + "local_dir = f\"sample_data/{raw_datasets}\"\n", + "os.makedirs(local_dir, exist_ok=True)\n", + "\n", + "r = requests.get(api_url)\n", + "r.raise_for_status()\n", + "\n", + "for item in r.json():\n", + " if item[\"name\"].endswith(\".csv\"):\n", + " print(f\"Downloading {item['name']}...\")\n", + " file_data = requests.get(item[\"download_url\"])\n", + " with open(os.path.join(local_dir, item[\"name\"]), \"wb\") as f:\n", + " f.write(file_data.content)\n", + "\n", + "print(\"All CSV files downloaded successfully.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Build the Semantic Model\n", + "\n", + "Conceptual Search operates on top of a semantic layer. Before we can use it, we need to build this layer using the `SemanticModel` class. This process involves profiling the data, predicting links between tables, and generating a business glossary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_config(table_name: str) -> str:\n", + " \"\"\"Append the base URL to the table name.\"\"\"\n", + " return {\n", + " \"path\": f\"./sample_data/healthcare/{table_name}.csv\",\n", + " \"type\": \"csv\",\n", + " }\n", + "\n", + "\n", + "table_names = [\n", + " \"allergies\",\n", + " \"careplans\",\n", + " \"claims\",\n", + " \"claims_transactions\",\n", + " \"conditions\",\n", + " \"devices\",\n", + " \"encounters\",\n", + " \"imaging_studies\",\n", + " \"immunizations\",\n", + " \"medications\",\n", + " \"observations\",\n", + " \"organizations\",\n", + " \"patients\",\n", + " \"payers\",\n", + " \"payer_transitions\",\n", + " \"procedures\",\n", + " \"providers\",\n", + " \"supplies\",\n", + "]\n", + "\n", + "datasets = {table: generate_config(table) for table in table_names}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Generate a Data Product Plan\n", + "\n", + "Now that the semantic layer is built, we can use the `DataProduct` class to generate a plan from a natural language query. The `plan()` method kicks off the first stage of Conceptual Search." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from intugle import DataProduct\n", + "\n", + "dp = DataProduct()\n", + "\n", + "# Generate a plan from a natural language query\n", + "query = \"patient 360 view\"\n", + "data_product_plan = await dp.plan(query=query)\n", + "data_product_plan" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Modify the Plan\n", + "\n", + "The generated plan is a starting point. You can programmatically modify it to refine the final output. Let's rename an attribute and disable another one that we don't need." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's assume the plan included 'Patient Last Name' and we want to rename it\n", + "data_product_plan.rename_attribute('Patient Last Name', 'Family Name')\n", + "\n", + "# Let's also assume it included 'Patient Birth Date' which we don't need\n", + "data_product_plan.disable_attribute('Patient Birth Date')\n", + "\n", + "print(\"--- Modified Plan ---\")\n", + "data_product_plan" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Build the Data Product\n", + "\n", + "Once you're satisfied with the plan, you can proceed to the building stage. This will trigger the second AI agent to map the plan's attributes to physical columns and generate the final SQL query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Build the data product from the modified plan\n", + "data_product = await dp.build_from_plan(data_product_plan)\n", + "\n", + "# Access the results as a pandas DataFrame\n", + "df = data_product.to_df()\n", + "print(df.head())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also inspect the final, generated SQL query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(data_product.sql_query)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "You have successfully used Conceptual Search to:\n", + "1. Generate a data product plan from a natural language query.\n", + "2. Review and modify the AI-generated plan.\n", + "3. Build a unified data product without writing any SQL." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/pyproject.toml b/pyproject.toml index 1abcb8c..9a443cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ ] dependencies = [ - "langchain-community>=0.3.21,<=0.3.30", - "langchain-openai>=0.3.28,<=0.3.33", + "langchain-community>=0.3.21,<1.0.0", + "langchain-openai>=0.3.28,<1.0.0", "langgraph>=0.6.4", "nltk>=3.9.1", "numpy<=2.3.0", diff --git a/test_col_search.py b/test_col_search.py deleted file mode 100644 index 4a82546..0000000 --- a/test_col_search.py +++ /dev/null @@ -1,24 +0,0 @@ - -# import asyncio -import asyncio - -from intugle.core import settings -from intugle.core.conceptual_search.graph_based_column_search.networkx_initializers import prepare_networkx_graph -from intugle.core.conceptual_search.graph_based_column_search.retreiver import GraphSearch -from intugle.parser.manifest import ManifestLoader - -manifest_loader = ManifestLoader(settings.MODELS_DIR) -manifest_loader.load() -manifest = manifest_loader.manifest - -prepare_networkx_graph(manifest) - -graph = GraphSearch() - -while True: - print("\n\nEnter query: ") - query = input() - if query == "q": - break - results = asyncio.run(graph.get_shortlisted_columns(query=query)) - print(results.sort_values(by="score", ascending=False)) diff --git a/test_table_search.py b/test_table_search.py deleted file mode 100644 index bd7d931..0000000 --- a/test_table_search.py +++ /dev/null @@ -1,25 +0,0 @@ - -# import asyncio -import asyncio - -from intugle.core import settings -from intugle.core.conceptual_search.graph_based_table_search.networkx_initializers import prepare_networkx_graph -from intugle.core.conceptual_search.graph_based_table_search.retreiver import GraphSearch -from intugle.parser.manifest import ManifestLoader - -manifest_loader = ManifestLoader(settings.MODELS_DIR) -manifest_loader.load() -manifest = manifest_loader.manifest - -prepare_networkx_graph(manifest) - -graph = GraphSearch() - -while True: - print("\n\nEnter query: ") - query = input() - if query == "q": - break - results = asyncio.run(graph.get_shortlisted_tables(query=query)) - print(results.sort_values(by="score", ascending=False)) - # breakpoint() \ No newline at end of file diff --git a/uv.lock b/uv.lock index 5d979a9..2702500 100644 --- a/uv.lock +++ b/uv.lock @@ -1890,8 +1890,8 @@ requires-dist = [ { name = "fastapi", extras = ["standard"], specifier = ">=0.116.1" }, { name = "graphviz", marker = "extra == 'streamlit'" }, { name = "langchain", extras = ["anthropic", "google-genai", "openai"], specifier = ">=0.3.27,<1.0.0" }, - { name = "langchain-community", specifier = ">=0.3.21,<=0.3.30" }, - { name = "langchain-openai", specifier = ">=0.3.28,<=0.3.33" }, + { name = "langchain-community", specifier = ">=0.3.21,<1.0.0" }, + { name = "langchain-openai", specifier = ">=0.3.28,<1.0.0" }, { name = "langgraph", specifier = ">=0.6.4" }, { name = "matplotlib", specifier = ">=3.10.5" }, { name = "mcp", extras = ["cli"], specifier = ">=1.12.4" }, From 891c8be7bed91fb727f04443f0a861057ba6977b Mon Sep 17 00:00:00 2001 From: Raphael Tony Date: Tue, 11 Nov 2025 17:18:36 +0530 Subject: [PATCH 26/26] upgraded version to 1.1.0 , updated notebook docs --- .../semantic-intelligence/semantic-search.md | 2 +- notebooks/quickstart_fmcg.ipynb | 2 +- notebooks/quickstart_fmcg_snowflake.ipynb | 8 ++++---- notebooks/quickstart_healthcare.ipynb | 2 +- notebooks/quickstart_healthcare_databricks.ipynb | 2 +- notebooks/quickstart_native_databricks.ipynb | 2 +- notebooks/quickstart_native_snowflake.ipynb | 2 +- notebooks/quickstart_sports_media.ipynb | 2 +- notebooks/quickstart_tech_manufacturing.ipynb | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docsite/docs/core-concepts/semantic-intelligence/semantic-search.md b/docsite/docs/core-concepts/semantic-intelligence/semantic-search.md index b02fc73..943bc2f 100644 --- a/docsite/docs/core-concepts/semantic-intelligence/semantic-search.md +++ b/docsite/docs/core-concepts/semantic-intelligence/semantic-search.md @@ -131,7 +131,7 @@ from intugle.semantic_search import SemanticSearch # This assumes your project's .yml files are in the default location. # You can also specify the path to your models directory: -# search_client = SemanticSearch(project_base="/path/to/your/models") +# search_client = SemanticSearch(models_dir_path="/path/to/your/models") search_client = SemanticSearch() # 1. Initialize the search index. diff --git a/notebooks/quickstart_fmcg.ipynb b/notebooks/quickstart_fmcg.ipynb index 27af7cb..5130ca2 100644 --- a/notebooks/quickstart_fmcg.ipynb +++ b/notebooks/quickstart_fmcg.ipynb @@ -2562,7 +2562,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_fmcg_snowflake.ipynb b/notebooks/quickstart_fmcg_snowflake.ipynb index 4c21511..0919191 100644 --- a/notebooks/quickstart_fmcg_snowflake.ipynb +++ b/notebooks/quickstart_fmcg_snowflake.ipynb @@ -1212,7 +1212,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "ce110000-1111-2222-3333-ffffff000024", "metadata": { "application/vnd.databricks.v1+cell": { @@ -1657,7 +1657,7 @@ " \"\"\"\n", " try:\n", " # 1. Load metadata from Intugle's YAML files\n", - " manifest = load_intugle_manifest(settings.PROJECT_BASE)\n", + " manifest = load_intugle_manifest(settings.MODELS_DIR)\n", "\n", " # 2. Create tables and populate them with data\n", " create_and_populate_tables_snowpark(\n", @@ -1674,7 +1674,7 @@ " apply_tags_snowpark(session, manifest, SNOWFLAKE_DATABASE, SNOWFLAKE_SCHEMA)\n", "\n", " # 5. Upload YAML files to Snowflake Stage\n", - " upload_folder_to_stage(session, settings.PROJECT_BASE, SNOWFLAKE_CSV_STAGE_NAME)\n", + " upload_folder_to_stage(session, settings.MODELS_DIR, SNOWFLAKE_CSV_STAGE_NAME)\n", "\n", " print(\"\\n-------------------------------------------------\")\n", " print(\"Snowflake setup process completed successfully!\")\n", @@ -1712,7 +1712,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_healthcare.ipynb b/notebooks/quickstart_healthcare.ipynb index 0b6539b..55c3fd8 100644 --- a/notebooks/quickstart_healthcare.ipynb +++ b/notebooks/quickstart_healthcare.ipynb @@ -4431,7 +4431,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_healthcare_databricks.ipynb b/notebooks/quickstart_healthcare_databricks.ipynb index 1e564a6..5379361 100644 --- a/notebooks/quickstart_healthcare_databricks.ipynb +++ b/notebooks/quickstart_healthcare_databricks.ipynb @@ -1713,7 +1713,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_native_databricks.ipynb b/notebooks/quickstart_native_databricks.ipynb index e6dc45e..5d52da1 100644 --- a/notebooks/quickstart_native_databricks.ipynb +++ b/notebooks/quickstart_native_databricks.ipynb @@ -1235,7 +1235,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_native_snowflake.ipynb b/notebooks/quickstart_native_snowflake.ipynb index fc0662b..5603af3 100644 --- a/notebooks/quickstart_native_snowflake.ipynb +++ b/notebooks/quickstart_native_snowflake.ipynb @@ -4949,7 +4949,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_sports_media.ipynb b/notebooks/quickstart_sports_media.ipynb index 34c41f1..404d2de 100644 --- a/notebooks/quickstart_sports_media.ipynb +++ b/notebooks/quickstart_sports_media.ipynb @@ -15365,7 +15365,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/notebooks/quickstart_tech_manufacturing.ipynb b/notebooks/quickstart_tech_manufacturing.ipynb index 339f8ca..507f00e 100644 --- a/notebooks/quickstart_tech_manufacturing.ipynb +++ b/notebooks/quickstart_tech_manufacturing.ipynb @@ -4857,7 +4857,7 @@ "\n", "The SemanticModel results are used to generate YAML files which are saved automatically. These files defines the semantic layer, including the models (tables) and their relationships. \n", "\n", - "By default, these files are saved in the current working directory. You can configure this path by setting the `PROJECT_BASE` environment variable." + "By default, these files are saved in the current working directory under `intugle/models`. You can configure this path by setting the `MODELS_DIR` environment variable." ] }, { diff --git a/pyproject.toml b/pyproject.toml index 9a443cc..c83fd48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "intugle" -version = "1.0.13" +version = "1.1.0" authors = [ { name="Intugle", email="hello@intugle.ai" }, ] diff --git a/uv.lock b/uv.lock index 2702500..76d04e2 100644 --- a/uv.lock +++ b/uv.lock @@ -1787,7 +1787,7 @@ wheels = [ [[package]] name = "intugle" -version = "1.0.13" +version = "1.1.0" source = { editable = "." } dependencies = [ { name = "aiofiles" },