[DayZ Parameters] 'missing_values_proportion' should not be valid for any key columns (#2709)

Co-authored-by: R-Palazzo <[email protected]>

Author: frances-h

sdv/single_table/dayz.py

@@ -132,7 +132,7 @@ def _validate_categorical_parameters(column_parameters, column_table_msg):
         raise SynthesizerProcessingError(msg)
 
 
-def _validate_missing_value_parameters(column_parameters, column_table_msg):
+def _validate_missing_value_parameters(column_parameters, column_table_msg, is_key_column):
     missing_values_proportion = column_parameters['missing_values_proportion']
     if not _is_numerical(missing_values_proportion) or (
         missing_values_proportion < 0.0 or missing_values_proportion > 1.0
@@ -142,9 +142,15 @@ def _validate_missing_value_parameters(column_parameters, column_table_msg):
             'must be a float between 0.0 and 1.0.'
         )
         raise SynthesizerProcessingError(msg)
+    elif is_key_column and missing_values_proportion != 0:
+        msg = (
+            f"Invalid 'missing_values_proportion' parameter for {column_table_msg}. Primary "
+            "and alternate keys must have 'missing_values_proportion' parameter set to zero."
+        )
+        raise SynthesizerProcessingError(msg)
 
 
-def _validate_column_parameters(table, column, column_metadata, column_parameters):
+def _validate_column_parameters(table, column, column_metadata, column_parameters, is_key_column):
     column_table_msg = f"column '{column}' in table '{table}'"
     sdtype = column_metadata['sdtype']
     sdtype_parameters = SDTYPE_TO_PARAMETERS.get(sdtype, COLUMN_PARAMETER_KEYS)
@@ -165,7 +171,7 @@ def _validate_column_parameters(table, column, column_metadata, column_parameter
         _validate_categorical_parameters(column_parameters, column_table_msg)
 
     if 'missing_values_proportion' in column_parameters:
-        _validate_missing_value_parameters(column_parameters, column_table_msg)
+        _validate_missing_value_parameters(column_parameters, column_table_msg, is_key_column)
 
 
 def _validate_table_parameters(table, table_metadata, table_parameters):
@@ -186,9 +192,11 @@ def _validate_table_parameters(table, table_metadata, table_parameters):
         )
         raise SynthesizerProcessingError(msg)
 
+    key_columns = table_metadata._get_primary_and_alternate_keys()
     for column, column_parameters in table_parameters.get('columns', {}).items():
+        is_key_column = column in key_columns
         _validate_column_parameters(
-            table, column, table_metadata.columns[column], column_parameters
+            table, column, table_metadata.columns[column], column_parameters, is_key_column
         )
 
 

tests/unit/single_table/test__dayz_utils.py

@@ -27,20 +27,26 @@ def test_detect_column_parameter():
     """Test the `detect_column_parameters` method."""
     # Setup
     data = pd.DataFrame({
+        'pk': [0, 1, 2, 3],
         'num_col': [1.0, 2.5, 3.0, None],
         'cat_col': ['A', 'B', 'A', None],
         'date_col': ['2020-01-01', '2020-01-02', None, None],
         'date_col_2': ['2020 Jan 01', '2020 Jan 02', '2020 Jan 03', None],
+        'alt_key': ['id0', 'id1', 'id2', 'id3'],
     })
     metadata = Metadata.load_from_dict({
         'tables': {
             'table_name': {
                 'columns': {
+                    'pk': {'sdtype': 'id'},
                     'num_col': {'sdtype': 'numerical'},
                     'cat_col': {'sdtype': 'categorical'},
                     'date_col': {'sdtype': 'datetime', 'datetime_format': '%Y-%m-%d'},
                     'date_col_2': {'sdtype': 'datetime'},
-                }
+                    'alt_key': {'sdtype': 'ssn'},
+                },
+                'primary_key': 'pk',
+                'alternate_keys': ['alt_key'],
             }
         }
     })
@@ -50,6 +56,7 @@ def test_detect_column_parameter():
     # Assert
     assert result == {
         'columns': {
+            'pk': {'missing_values_proportion': 0.0},
             'num_col': {
                 'num_decimal_digits': 1,
                 'min_value': 1.0,
@@ -70,6 +77,7 @@ def test_detect_column_parameter():
                 'end_timestamp': '2020-01-03 00:00:00',
                 'missing_values_proportion': 0.25,
             },
+            'alt_key': {'missing_values_proportion': 0.0},
         }
     }
 

tests/unit/single_table/test_dayz.py

@@ -28,7 +28,8 @@ def metadata():
                     'categorical': {'sdtype': 'categorical'},
                     'pii': {'sdtype': 'ssn'},
                     'extra_column': {'sdtype': 'numerical'},
-                }
+                },
+                'primary_key': 'id',
             }
         }
     })
@@ -97,21 +98,31 @@ def test__validate_column_parameter():
     column_metadata = {'sdtype': 'id'}
     bad_column_parameters = {'invalid_key': None}
     bad_missing_value = {'missing_values_proportion': 100}
+    bad_key_missing_value = {'missing_values_proportion': 0.5}
 
     # Run and Assert
     expected_bad_column_msg = re.escape(
         "The parameters for column 'column' in table 'table' contains unexpected "
         "key(s) 'invalid_key'."
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_column_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_column_parameters)
+        _validate_column_parameters(
+            'table', 'column', column_metadata, bad_column_parameters, False
+        )
 
     expected_bad_missing_value_msg = re.escape(
         "The 'missing_values_proportion' parameter for column 'column' in table 'table' "
         'must be a float between 0.0 and 1.0.'
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_missing_value_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_missing_value)
+        _validate_column_parameters('table', 'column', column_metadata, bad_missing_value, False)
+
+    expected_missing_values_with_key_msg = re.escape(
+        "Invalid 'missing_values_proportion' parameter for column 'column' in table 'table'. "
+        "Primary and alternate keys must have 'missing_values_proportion' parameter set to zero."
+    )
+    with pytest.raises(SynthesizerProcessingError, match=expected_missing_values_with_key_msg):
+        _validate_column_parameters('table', 'column', column_metadata, bad_key_missing_value, True)
 
 
 def test__validate_column_parameters_numerical():
@@ -127,21 +138,25 @@ def test__validate_column_parameters_numerical():
         "The 'min_value' parameter for column 'column' in table 'table' must be a float."
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_parameter_value_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_parameter_value)
+        _validate_column_parameters('table', 'column', column_metadata, bad_parameter_value, False)
 
     expected_bad_min_max_msg = re.escape(
         "Invalid parameters for column 'column' in table 'table'. The 'min_value' "
         "must be less than or equal to the 'max_value'"
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_min_max_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_min_max_combination)
+        _validate_column_parameters(
+            'table', 'column', column_metadata, bad_min_max_combination, False
+        )
 
     expected_bad_num_decimal_digits_msg = re.escape(
         "The 'num_decimal_digits' parameter for column 'column' in table 'table' must be an "
         'integer greater than or equal to zero.'
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_num_decimal_digits_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_num_decimal_digits)
+        _validate_column_parameters(
+            'table', 'column', column_metadata, bad_num_decimal_digits, False
+        )
 
 
 def test__validate_column_parameters_datetime():
@@ -157,28 +172,32 @@ def test__validate_column_parameters_datetime():
         "The 'start_timestamp' parameter for column 'column' in table 'table' must be a string."
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_parameter_value_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_parameter_value)
+        _validate_column_parameters('table', 'column', column_metadata, bad_parameter_value, False)
 
     expected_bad_datetime_value_msg = re.escape(
         "The 'start_timestamp' parameter for column 'column' in table 'table' is not a valid "
         'datetime string or does not match the date time format (%d %b %Y).'
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_datetime_value_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_datetime_value)
+        _validate_column_parameters('table', 'column', column_metadata, bad_datetime_value, False)
 
     expected_bad_value_no_format_msg = re.escape(
         "The 'start_timestamp' parameter for column 'column' in table 'table' is not a "
         'valid datetime string.'
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_value_no_format_msg):
-        _validate_column_parameters('table', 'column', {'sdtype': 'datetime'}, bad_datetime_value)
+        _validate_column_parameters(
+            'table', 'column', {'sdtype': 'datetime'}, bad_datetime_value, False
+        )
 
     expected_bad_start_end_msg = re.escape(
         "Invalid parameters for column 'column' in table 'table'. The 'start_timestamp' "
         "must be less than the 'end_timestamp'"
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_bad_start_end_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_start_end_combination)
+        _validate_column_parameters(
+            'table', 'column', column_metadata, bad_start_end_combination, False
+        )
 
 
 def test__validate_column_parameters_categorical():
@@ -192,7 +211,7 @@ def test__validate_column_parameters_categorical():
         "The 'category_values' parameter for column 'column' in table 'table' must be a list."
     )
     with pytest.raises(SynthesizerProcessingError, match=expected_msg):
-        _validate_column_parameters('table', 'column', column_metadata, bad_category_values)
+        _validate_column_parameters('table', 'column', column_metadata, bad_category_values, False)
 
 
 @patch('sdv.single_table.dayz._validate_column_parameters')
@@ -202,6 +221,7 @@ def test__validate_table_parameters(mock__validate_column_parameters, metadata,
     table_metadata = metadata.tables['table']
     bad_table_columns = {'columns': {'bad_column': {}}}
     bad_num_rows = {'num_rows': -1}
+    keys = ['id']
 
     # Run and Assert
     expected_bad_column_msg = re.escape(
@@ -222,7 +242,7 @@ def test__validate_table_parameters(mock__validate_column_parameters, metadata,
 
     # Assert
     expected_calls = [
-        call('table', col, table_metadata.columns[col], col_parameters)
+        call('table', col, table_metadata.columns[col], col_parameters, col in keys)
         for col, col_parameters in dayz_parameters['tables']['table']['columns'].items()
     ]
     mock__validate_column_parameters.assert_has_calls(expected_calls)