diff --git a/docsite/docs/core-concepts/dataset.md b/docsite/docs/core-concepts/dataset.md index 5eefaa7..31a8d15 100644 --- a/docsite/docs/core-concepts/dataset.md +++ b/docsite/docs/core-concepts/dataset.md @@ -23,33 +23,29 @@ The `DataSet` uses a system of **Adapters** under the hood to connect to differe ### Centralized Metadata -All analysis results for a data source are stored within the `dataset.source_table_model` attribute. This attribute is a structured Pydantic model that makes accessing metadata predictable and easy. - -#### Metadata Structure - -The metadata is organized using the following Pydantic models: - -- `SourceTables`: The root object, containing table-level information. - - `name: str` - - `description: str` - - `key: Optional[str]` - - `columns: List[Column]` - - `profiling_metrics: Optional[ModelProfilingMetrics]` -- `Column`: Contains metadata for a single column. - - `name: str` - - `description: Optional[str]` - - `type: Optional[str]` (e.g., 'integer', 'date') - - `category: Literal["dimension", "measure"]` - - `tags: Optional[List[str]]` - - `profiling_metrics: Optional[ColumnProfilingMetrics]` -- `ColumnProfilingMetrics`: Detailed statistics for a column. +All analysis results for a data source are stored within the `dataset.source_table_model` attribute. This attribute is a structured Pydantic model that makes accessing metadata predictable and easy. For more convenient access to column-level data, the `DataSet` also provides a `columns` dictionary. + +#### Metadata Structure and Access + +The metadata is organized using Pydantic models, but you can access it easily through the `DataSet`'s attributes. + +- **Table-Level Metadata**: Accessed via `dataset.source_table_model`. + - `.name: str` + - `.description: str` + - `.key: Optional[str]` +- **Column-Level Metadata**: Accessed via the `dataset.columns` dictionary, where keys are column names. + - `[column_name].description: Optional[str]` + - `[column_name].type: Optional[str]` (e.g., 'integer', 'date') + - `[column_name].category: Literal["dimension", "measure"]` + - `[column_name].tags: Optional[List[str]]` + - `[column_name].profiling_metrics: Optional[ColumnProfilingMetrics]` +- **`ColumnProfilingMetrics`**: Detailed statistics for a column. - `count: Optional[int]` - `null_count: Optional[int]` - `distinct_count: Optional[int]` + - `sample_data: Optional[List[Any]]` -#### Accessing Metadata - -You can access this rich metadata directly from the `DataSet` object. +#### Example of Accessing Metadata ```python # Assuming 'kb' is a built KnowledgeBuilder instance @@ -57,17 +53,15 @@ customers_dataset = kb.datasets['customers'] # Access table-level metadata print(f"Table Name: {customers_dataset.source_table_model.name}") -print(f"Table Description: {customers_dataset.source_table_model.description}") print(f"Primary Key: {customers_dataset.source_table_model.key}") -# Access column-level metadata for the first column -first_column = customers_dataset.source_table_model.columns[0] -print(f"Column Name: {first_column.name}") -print(f"Column Description: {first_column.description}") -print(f"Column Type: {first_column.type}") +# Access column-level metadata using the 'columns' dictionary +email_column = customers_dataset.columns['email'] +print(f"Column Name: {email_column.name}") +print(f"Column Description: {email_column.description}") # Access profiling metrics for that column -metrics = first_column.profiling_metrics +metrics = email_column.profiling_metrics if metrics: print(f"Distinct Count: {metrics.distinct_count}") ``` diff --git a/src/intugle/analysis/models.py b/src/intugle/analysis/models.py index 8564ad4..714bd05 100644 --- a/src/intugle/analysis/models.py +++ b/src/intugle/analysis/models.py @@ -46,7 +46,7 @@ def __init__(self, data: DataSetData, name: str): # A dictionary to store the results of each analysis step self.source_table_model: SourceTables = SourceTables(name=name, description="") - self._columns_map: Dict[str, Column] = {} # A convenience map for quick column lookup + self.columns: Dict[str, Column] = {} # A convenience map for quick column lookup # Check if a YAML file exists and load it file_path = os.path.join(settings.PROJECT_BASE, f"{self.name}.yml") @@ -86,7 +86,7 @@ def _populate_from_yaml(self, yaml_data: dict): source = yaml_data.get("sources", [])[0] table = source.get("table", {}) self.source_table_model = SourceTables.model_validate(table) - self._columns_map = {col.name: col for col in self.source_table_model.columns} + self.columns = {col.name: col for col in self.source_table_model.columns} @property def sql_query(self): @@ -112,7 +112,7 @@ def profile_table(self) -> Self: self.source_table_model.profiling_metrics.count = table_profile.count self.source_table_model.columns = [Column(name=col_name) for col_name in table_profile.columns] - self._columns_map = {col.name: col for col in self.source_table_model.columns} + self.columns = {col.name: col for col in self.source_table_model.columns} return self def profile_columns(self) -> Self: @@ -163,7 +163,7 @@ def identify_datatypes_l1(self) -> "DataSet": column_datatypes_l1 = [DataTypeIdentificationL1Output(**row) for row in l1_result.to_dict(orient="records")] for col_l1 in column_datatypes_l1: - self._columns_map[col_l1.column_name].type = col_l1.datatype_l1 + self.columns[col_l1.column_name].type = col_l1.datatype_l1 return self def identify_datatypes_l2(self) -> "DataSet": @@ -191,7 +191,7 @@ def identify_datatypes_l2(self) -> "DataSet": column_datatypes_l2 = [DataTypeIdentificationL2Output(**row) for row in l2_result.to_dict(orient="records")] for col_l2 in column_datatypes_l2: - self._columns_map[col_l2.column_name].category = col_l2.datatype_l2 + self.columns[col_l2.column_name].category = col_l2.datatype_l2 return self def identify_keys(self, save: bool = False) -> Self: @@ -291,7 +291,7 @@ def generate_glossary(self, domain: str = "", save: bool = False) -> Self: self.source_table_model.description = table_glossary for _, row in glossary_df.iterrows(): - column = self._columns_map[row["column_name"]] + column = self.columns[row["column_name"]] column.description = row.get("business_glossary", "") column.tags = row.get("business_tags", []) @@ -345,8 +345,12 @@ def load_from_yaml(self, file_path: str) -> None: if not self._is_yaml_stale(yaml_data): self._populate_from_yaml(yaml_data) - def reload_from_yaml(self, file_path: str) -> None: + def reload_from_yaml(self, file_path: Optional[str] = None) -> None: """Forces a reload from a YAML file, bypassing staleness checks.""" + if file_path is None: + file_path = f"{self.name}.yml" + file_path = os.path.join(settings.PROJECT_BASE, file_path) + with open(file_path, "r") as f: yaml_data = yaml.safe_load(f) self._populate_from_yaml(yaml_data) diff --git a/tests/analysis/test_business_glossary.py b/tests/analysis/test_business_glossary.py index 6e36cda..0566949 100644 --- a/tests/analysis/test_business_glossary.py +++ b/tests/analysis/test_business_glossary.py @@ -42,7 +42,7 @@ def test_business_glossary_generator(): assert len(dataset.source_table_model.columns) == len(df.columns) # Check a specific column's glossary entry - product_id_column = dataset._columns_map.get("product_id") + product_id_column = dataset.columns.get("product_id") assert product_id_column is not None assert product_id_column.description is not None assert len(product_id_column.description) > 0 diff --git a/tests/analysis/test_datatype_identification.py b/tests/analysis/test_datatype_identification.py index dc86013..ba3ed2c 100644 --- a/tests/analysis/test_datatype_identification.py +++ b/tests/analysis/test_datatype_identification.py @@ -26,7 +26,7 @@ def test_datatype_identification_l1_end_to_end(): analysis_results = pipeline.run(COMPLEX_DF, DF_NAME) # Check the final output of the L1 step - columns_map = analysis_results._columns_map + columns_map = analysis_results.columns assert columns_map['user_id'].type == 'integer' assert columns_map['product_name'].type == 'close_ended_text' assert columns_map['price'].type == 'float' @@ -46,7 +46,7 @@ def test_datatype_identification_l2_end_to_end(): analysis_results = pipeline.run(COMPLEX_DF, DF_NAME) # Check the final output of the L2 step - columns_map = analysis_results._columns_map + columns_map = analysis_results.columns assert columns_map['user_id'].category == 'dimension' assert columns_map['product_name'].category == 'dimension' assert columns_map['price'].category == 'measure' diff --git a/tests/analysis/test_pipeline.py b/tests/analysis/test_pipeline.py index c884738..f7a3b3c 100644 --- a/tests/analysis/test_pipeline.py +++ b/tests/analysis/test_pipeline.py @@ -44,7 +44,7 @@ def test_column_profiling_with_complex_data(): ColumnProfiler() ]) analysis_results = pipeline.run(COMPLEX_DF, DF_NAME) - columns_map = analysis_results._columns_map + columns_map = analysis_results.columns assert len(columns_map) == 5 # --- Assertions for 'user_id' ---