added default filepath to reload, renamed columns, updated documentation

docsite/docs/core-concepts/dataset.md

@@ -23,51 +23,45 @@ The `DataSet` uses a system of **Adapters** under the hood to connect to differe
 
 ### Centralized Metadata
 
-All analysis results for a data source are stored within the `dataset.source_table_model` attribute. This attribute is a structured Pydantic model that makes accessing metadata predictable and easy.
-
-#### Metadata Structure
-
-The metadata is organized using the following Pydantic models:
-
--   `SourceTables`: The root object, containing table-level information.
-    -   `name: str`
-    -   `description: str`
-    -   `key: Optional[str]`
-    -   `columns: List[Column]`
-    -   `profiling_metrics: Optional[ModelProfilingMetrics]`
--   `Column`: Contains metadata for a single column.
-    -   `name: str`
-    -   `description: Optional[str]`
-    -   `type: Optional[str]` (e.g., 'integer', 'date')
-    -   `category: Literal["dimension", "measure"]`
-    -   `tags: Optional[List[str]]`
-    -   `profiling_metrics: Optional[ColumnProfilingMetrics]`
--   `ColumnProfilingMetrics`: Detailed statistics for a column.
+All analysis results for a data source are stored within the `dataset.source_table_model` attribute. This attribute is a structured Pydantic model that makes accessing metadata predictable and easy. For more convenient access to column-level data, the `DataSet` also provides a `columns` dictionary.
+
+#### Metadata Structure and Access
+
+The metadata is organized using Pydantic models, but you can access it easily through the `DataSet`'s attributes.
+
+-   **Table-Level Metadata**: Accessed via `dataset.source_table_model`.
+    -   `.name: str`
+    -   `.description: str`
+    -   `.key: Optional[str]`
+-   **Column-Level Metadata**: Accessed via the `dataset.columns` dictionary, where keys are column names.
+    -   `[column_name].description: Optional[str]`
+    -   `[column_name].type: Optional[str]` (e.g., 'integer', 'date')
+    -   `[column_name].category: Literal["dimension", "measure"]`
+    -   `[column_name].tags: Optional[List[str]]`
+    -   `[column_name].profiling_metrics: Optional[ColumnProfilingMetrics]`
+-   **`ColumnProfilingMetrics`**: Detailed statistics for a column.
     -   `count: Optional[int]`
     -   `null_count: Optional[int]`
     -   `distinct_count: Optional[int]`
+    -   `sample_data: Optional[List[Any]]`
 
-#### Accessing Metadata
-
-You can access this rich metadata directly from the `DataSet` object.
+#### Example of Accessing Metadata
 
 ```python
 # Assuming 'kb' is a built KnowledgeBuilder instance
 customers_dataset = kb.datasets['customers']
 
 # Access table-level metadata
 print(f"Table Name: {customers_dataset.source_table_model.name}")
-print(f"Table Description: {customers_dataset.source_table_model.description}")
 print(f"Primary Key: {customers_dataset.source_table_model.key}")
 
-# Access column-level metadata for the first column
-first_column = customers_dataset.source_table_model.columns[0]
-print(f"Column Name: {first_column.name}")
-print(f"Column Description: {first_column.description}")
-print(f"Column Type: {first_column.type}")
+# Access column-level metadata using the 'columns' dictionary
+email_column = customers_dataset.columns['email']
+print(f"Column Name: {email_column.name}")
+print(f"Column Description: {email_column.description}")
 
 # Access profiling metrics for that column
-metrics = first_column.profiling_metrics
+metrics = email_column.profiling_metrics
 if metrics:
     print(f"Distinct Count: {metrics.distinct_count}")
 ```

src/intugle/analysis/models.py

@@ -46,7 +46,7 @@ def __init__(self, data: DataSetData, name: str):
 
         # A dictionary to store the results of each analysis step
         self.source_table_model: SourceTables = SourceTables(name=name, description="")
-        self._columns_map: Dict[str, Column] = {}  # A convenience map for quick column lookup
+        self.columns: Dict[str, Column] = {}  # A convenience map for quick column lookup
 
         # Check if a YAML file exists and load it
         file_path = os.path.join(settings.PROJECT_BASE, f"{self.name}.yml")
@@ -86,7 +86,7 @@ def _populate_from_yaml(self, yaml_data: dict):
         source = yaml_data.get("sources", [])[0]
         table = source.get("table", {})
         self.source_table_model = SourceTables.model_validate(table)
-        self._columns_map = {col.name: col for col in self.source_table_model.columns}
+        self.columns = {col.name: col for col in self.source_table_model.columns}
 
     @property
     def sql_query(self):
@@ -112,7 +112,7 @@ def profile_table(self) -> Self:
         self.source_table_model.profiling_metrics.count = table_profile.count
 
         self.source_table_model.columns = [Column(name=col_name) for col_name in table_profile.columns]
-        self._columns_map = {col.name: col for col in self.source_table_model.columns}
+        self.columns = {col.name: col for col in self.source_table_model.columns}
         return self
 
     def profile_columns(self) -> Self:
@@ -163,7 +163,7 @@ def identify_datatypes_l1(self) -> "DataSet":
         column_datatypes_l1 = [DataTypeIdentificationL1Output(**row) for row in l1_result.to_dict(orient="records")]
 
         for col_l1 in column_datatypes_l1:
-            self._columns_map[col_l1.column_name].type = col_l1.datatype_l1
+            self.columns[col_l1.column_name].type = col_l1.datatype_l1
         return self
 
     def identify_datatypes_l2(self) -> "DataSet":
@@ -191,7 +191,7 @@ def identify_datatypes_l2(self) -> "DataSet":
         column_datatypes_l2 = [DataTypeIdentificationL2Output(**row) for row in l2_result.to_dict(orient="records")]
 
         for col_l2 in column_datatypes_l2:
-            self._columns_map[col_l2.column_name].category = col_l2.datatype_l2
+            self.columns[col_l2.column_name].category = col_l2.datatype_l2
         return self
 
     def identify_keys(self, save: bool = False) -> Self:
@@ -291,7 +291,7 @@ def generate_glossary(self, domain: str = "", save: bool = False) -> Self:
         self.source_table_model.description = table_glossary
 
         for _, row in glossary_df.iterrows():
-            column = self._columns_map[row["column_name"]]
+            column = self.columns[row["column_name"]]
             column.description = row.get("business_glossary", "")
             column.tags = row.get("business_tags", [])
 
@@ -345,8 +345,12 @@ def load_from_yaml(self, file_path: str) -> None:
         if not self._is_yaml_stale(yaml_data):
             self._populate_from_yaml(yaml_data)
 
-    def reload_from_yaml(self, file_path: str) -> None:
+    def reload_from_yaml(self, file_path: Optional[str] = None) -> None:
         """Forces a reload from a YAML file, bypassing staleness checks."""
+        if file_path is None:
+            file_path = f"{self.name}.yml"
+        file_path = os.path.join(settings.PROJECT_BASE, file_path)
+
         with open(file_path, "r") as f:
             yaml_data = yaml.safe_load(f)
         self._populate_from_yaml(yaml_data)

tests/analysis/test_business_glossary.py

@@ -42,7 +42,7 @@ def test_business_glossary_generator():
     assert len(dataset.source_table_model.columns) == len(df.columns)
 
     # Check a specific column's glossary entry
-    product_id_column = dataset._columns_map.get("product_id")
+    product_id_column = dataset.columns.get("product_id")
     assert product_id_column is not None
     assert product_id_column.description is not None
     assert len(product_id_column.description) > 0

tests/analysis/test_datatype_identification.py

@@ -26,7 +26,7 @@ def test_datatype_identification_l1_end_to_end():
     analysis_results = pipeline.run(COMPLEX_DF, DF_NAME)
 
     # Check the final output of the L1 step
-    columns_map = analysis_results._columns_map
+    columns_map = analysis_results.columns
     assert columns_map['user_id'].type == 'integer'
     assert columns_map['product_name'].type == 'close_ended_text'
     assert columns_map['price'].type == 'float'
@@ -46,7 +46,7 @@ def test_datatype_identification_l2_end_to_end():
     analysis_results = pipeline.run(COMPLEX_DF, DF_NAME)
 
     # Check the final output of the L2 step
-    columns_map = analysis_results._columns_map
+    columns_map = analysis_results.columns
     assert columns_map['user_id'].category == 'dimension'
     assert columns_map['product_name'].category == 'dimension'
     assert columns_map['price'].category == 'measure'

tests/analysis/test_pipeline.py

@@ -44,7 +44,7 @@ def test_column_profiling_with_complex_data():
         ColumnProfiler()
     ])
     analysis_results = pipeline.run(COMPLEX_DF, DF_NAME)
-    columns_map = analysis_results._columns_map
+    columns_map = analysis_results.columns
     assert len(columns_map) == 5
 
     # --- Assertions for 'user_id' ---