Try pass in distance array to linkage method

Author: larsevj

src/ert/analysis/misfit_preprocessor.py

@@ -56,10 +56,12 @@ def cluster_responses(
     Cluster responses using hierarchical clustering based on Spearman correlation.
     Observations that tend to vary similarly across different simulation runs will be clustered together.
     """
-    correlation = spearmanr(responses).statistic
-    if isinstance(correlation, np.float64):
-        correlation = np.array([[1, correlation], [correlation, 1]])
-    linkage_matrix = linkage(correlation, "average", "euclidean")
+    distance = 1 - np.abs(spearmanr(responses).statistic)
+    if isinstance(distance, np.float64):
+        distance = np.array(distance)
+    linkage_matrix = linkage(
+        distance[np.triu_indices(distance.shape[0], k=1)], "average", "euclidean"
+    )
     return fcluster(linkage_matrix, nr_clusters, criterion="maxclust", depth=2)
 
 

tests/ert/unit_tests/analysis/test_es_update.py

@@ -59,7 +59,6 @@ def obs() -> polars.DataFrame:
 
 
 @pytest.mark.integration_test
-@pytest.mark.flaky(reruns=5)
 @pytest.mark.parametrize(
     "misfit_preprocess", [[["*"]], [], [["FOPR"]], [["FOPR"], ["WOPR_OP1_1*"]]]
 )

tests/ert/unit_tests/analysis/test_misfit_preprocessor.py

@@ -68,11 +68,11 @@ def test_misfit_preprocessor(nr_observations):
     nr_realizations = 1000
     Y = np.ones((nr_observations, nr_realizations))
     parameters_a = rng.standard_normal(nr_realizations)
-    parameters_b = rng.standard_normal(nr_realizations)
+    parameters_b = rng.normal(scale=5, size=nr_realizations)
     for i in range(nr_observations - 1):
         Y[i] = i + 1 * parameters_a
     Y[-1] = 5 + 1 * parameters_b
-    obs_errors = Y.mean(axis=1)
+    obs_errors = np.array([0.1] * nr_observations)
     Y_original = Y.copy()
     obs_error_copy = obs_errors.copy()
     result, *_ = main(Y, obs_errors)