Add Everest storage (port of seba_sqlite logic)

docs/everest/cli.rst

@@ -52,78 +52,7 @@ Using again the command `everest monitor config_file.yml`, will reattach to the
 Everest `export`
 ================
 
-.. argparse::
-    :module: everest.bin.everexport_script
-    :func: _build_args_parser
-    :prog: everexport_entry
-
-
-The everest export functionality is configured in the export section of the config file.
-The following represents an export section a config file set with default values.
-
-.. code-block:: yaml
-
-    export:
-        skip_export: False
-        keywords:
-        batches:
-        discard_gradient: True # Export only non-gradient simulations
-        discard_rejected: True # Export only increased merit simulations
-        csv_output_filepath: everest_output_folder/config_file.csv
-
-When the export command `everest export config_file.yml` is run with a config file that does not define an export section default values will be used, a `config_file.csv` file in the Everest output folder will be created.
-By default Everest exports only non-gradient with increased merit simulations when no config section is defined in the config file.
-The file will contain optimization data for all the optimization batches and the available eclipse keywords (if a data file is available) for only the non-gradient simulations and the simulations that increase merit.
-
-**Examples**
-
-* Export only non-gradient simulation using the following export section in the config file
-
-.. code-block:: yaml
-
-    export:
-        discard_rejected: False
-
-* Export only increased merit simulation using the following export section in the config file
-
-.. code-block:: yaml
-
-    export:
-        discard_gradient: False
-
-
-* Export only a list of available batches even if they are gradient batches and if no export section is defined.
-
- everest export config_file.yml --batches 0 2 4
-
-The command above is equivalent to having the following export section defined in the config file `config_file.yml`.
-
-.. code-block:: yaml
-
-    export:
-      batches: [0, 2, 4]
-
-* Exporting just a specific list of eclipse keywords requires the following export section defined in the config file.
-
-.. code-block:: yaml
-
-    export:
-      keywords: ['FOIP', 'FOPT']
-
-* Skip export by adding the following section in the config file.
-
-.. code-block:: yaml
-
-    export:
-      skip_export: True
-
-* Export will also be skipped if an empty list of batches is defined in the export section.
-
-.. code-block:: yaml
-
-    export:
-      batches: []
-
+The everest export has been removed. All data is now always exported to the optimization output directory.
 
 ==============
 Everest `lint`

src/ert/run_models/everest_run_model.py

@@ -9,21 +9,18 @@
 import shutil
 from collections import defaultdict
 from collections.abc import Callable
-from dataclasses import dataclass
 from enum import IntEnum
 from pathlib import Path
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, Protocol
 
 import numpy as np
-import seba_sqlite.sqlite_storage
 from numpy import float64
 from numpy._typing import NDArray
 from ropt.enums import EventType, OptimizerExitCode
 from ropt.evaluator import EvaluatorContext, EvaluatorResult
 from ropt.plan import BasicOptimizer
 from ropt.plan import Event as OptimizerEvent
-from seba_sqlite import SqliteStorage
 from typing_extensions import TypedDict
 
 from _ert.events import EESnapshot, EESnapshotUpdate, Event
@@ -32,6 +29,7 @@
 from ert.runpaths import Runpaths
 from ert.storage import open_storage
 from everest.config import ControlConfig, ControlVariableGuessListConfig, EverestConfig
+from everest.everest_storage import EverestStorage, OptimalResult
 from everest.optimizer.everest2ropt import everest2ropt
 from everest.simulator.everest_to_ert import everest_to_ert_config
 from everest.strings import EVEREST
@@ -70,24 +68,6 @@ class OptimizerCallback(Protocol):
     def __call__(self) -> str | None: ...
 
 
-@dataclass
-class OptimalResult:
-    batch: int
-    controls: list[Any]
-    total_objective: float
-
-    @staticmethod
-    def from_seba_optimal_result(
-        o: seba_sqlite.sqlite_storage.OptimalResult | None = None,
-    ) -> OptimalResult | None:
-        if o is None:
-            return None
-
-        return OptimalResult(
-            batch=o.batch, controls=o.controls, total_objective=o.total_objective
-        )
-
-
 class EverestExitCode(IntEnum):
     COMPLETED = 1
     TOO_FEW_REALIZATIONS = 2
@@ -206,23 +186,16 @@ def run_experiment(
         # Initialize the ropt optimizer:
         optimizer = self._create_optimizer()
 
-        # The SqliteStorage object is used to store optimization results from
-        # Seba in an sqlite database. It reacts directly to events emitted by
-        # Seba and is not called by Everest directly. The stored results are
-        # accessed by Everest via separate SebaSnapshot objects.
-        # This mechanism is outdated and not supported by the ropt package. It
-        # is retained for now via the seba_sqlite package.
-        seba_storage = SqliteStorage(  # type: ignore
-            optimizer, self._everest_config.optimization_output_dir
+        self.ever_storage = EverestStorage(
+            output_dir=Path(self._everest_config.optimization_output_dir),
         )
+        self.ever_storage.observe_optimizer(optimizer)
 
         # Run the optimization:
         optimizer_exit_code = optimizer.run().exit_code
 
         # Extract the best result from the storage.
-        self._result = OptimalResult.from_seba_optimal_result(
-            seba_storage.get_optimal_result()  # type: ignore
-        )
+        self._result = self.ever_storage.get_optimal_result()
 
         if self._exit_code is None:
             match optimizer_exit_code:

src/everest/__init__.py

@@ -21,16 +21,10 @@
     __version__ = "0.0.0"
 
 from everest import detached, jobs, templates, util
-from everest.bin.utils import export_to_csv, export_with_progress
-from everest.export import MetaDataColumnNames, filter_data
 
 __author__ = "Equinor ASA and TNO"
 __all__ = [
-    "MetaDataColumnNames",
     "detached",
-    "export_to_csv",
-    "export_with_progress",
-    "filter_data",
     "jobs",
     "load",
     "templates",