feat: refactor for general data science

.gitignore

@@ -170,3 +170,4 @@ mlruns/
 # shell script
 *.out
 *.sh
+.aider*

rdagent/app/data_mining/conf.py

@@ -23,7 +23,7 @@ class MedBasePropSetting(BasePropSetting):
     runner: str = "rdagent.scenarios.data_mining.developer.model_runner.DMModelRunner"
     """Runner class"""
 
-    summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelHypothesisExperiment2Feedback"
+    summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10

rdagent/app/data_science/conf.py

@@ -0,0 +1,48 @@
+from rdagent.app.kaggle.conf import KaggleBasePropSetting
+from rdagent.core.conf import ExtendedSettingsConfigDict
+
+
+class DataScienceBasePropSetting(KaggleBasePropSetting):
+    model_config = ExtendedSettingsConfigDict(env_prefix="DS_", protected_namespaces=())
+
+    # Main components
+    ## Scen
+    scen: str = "rdagent.scenarios.data_science.scen.KaggleScen"
+    """Scenario class for data mining model"""
+
+    ## proposal
+    exp_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen"
+
+    # the two below should be used in ExpGen
+    # hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
+    # """Hypothesis generation class"""
+    #
+    # hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
+    # """Hypothesis to experiment class"""
+
+    ## dev/coder
+    data_loader_coder: str = "rdagent.components.coder.data_science.raw_data_loader.DataLoaderCoSTEER"
+    """Data Loader CoSTEER"""
+
+    # feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    # """Feature Coder class"""
+
+    # model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
+    # """Model Feature Selection Coder class"""
+
+    # model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    # """Model Coder class"""
+
+    ## dev/runner
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    ## feedback
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGExperiment2Feedback"
+    """Summarizer class"""
+
+
+DS_RD_SETTING = DataScienceBasePropSetting()

rdagent/app/data_science/debug.py

@@ -0,0 +1,7 @@
+import fire
+
+from rdagent.scenarios.data_science.debug.data import create_debug_data
+
+
+if __name__ == "__main__":
+    fire.Fire(create_debug_data)

rdagent/app/data_science/loop.py

@@ -0,0 +1,131 @@
+from pathlib import Path
+from typing import Any
+
+import fire
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.data_science.ensemble import EnsembleCoSTEER
+from rdagent.components.coder.data_science.feature import FeatureCoSTEER
+from rdagent.components.coder.data_science.model import ModelCoSTEER
+from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER
+from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.proposal import HypothesisFeedback
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback
+from rdagent.scenarios.data_science.dev.runner import DSRunner
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+
+
+class DataScienceRDLoop(RDLoop):
+    skip_loop_error = ()
+
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+
+        ### shared components in the workflow  # TODO: check if
+        knowledge_base = (
+            import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+            if PROP_SETTING.knowledge_base != ""
+            else None
+        )
+
+        # 1) task generation from scratch
+        # self.scratch_gen: tuple[HypothesisGen, Hypothesis2Experiment] = DummyHypothesisGen(scen),
+
+        # 2) task generation from a complete solution
+        # self.exp_gen: ExpGen = import_class(PROP_SETTING.exp_gen)(scen)
+        self.exp_gen = DSExpGen(scen)
+        self.data_loader_coder = DataLoaderCoSTEER(scen)
+        self.feature_coder = FeatureCoSTEER(scen)
+        self.model_coder = ModelCoSTEER(scen)
+        self.ensemble_coder = EnsembleCoSTEER(scen)
+        self.workflow_coder = WorkflowCoSTEER(scen)
+
+        self.runner = DSRunner(scen)
+        # self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+        # logger.log_object(self.summarizer, tag="summarizer")
+
+        # self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+        self.trace = DSTrace(scen=scen)
+        self.summarizer = DSExperiment2Feedback(scen)
+        super(RDLoop, self).__init__()
+
+    def direct_exp_gen(self, prev_out: dict[str, Any]):
+        exp = self.exp_gen.gen(self.trace)
+        return exp
+
+    def coding(self, prev_out: dict[str, Any]):
+        exp: DSExperiment = prev_out["direct_exp_gen"]
+        if exp.hypothesis.component == "DataLoadSpec":
+            exp = self.data_loader_coder.develop(exp)
+        elif exp.hypothesis.component == "FeatureEng":
+            exp = self.feature_coder.develop(exp)
+        elif exp.hypothesis.component == "Model":
+            exp = self.model_coder.develop(exp)
+        elif exp.hypothesis.component == "Ensemble":
+            exp = self.ensemble_coder.develop(exp)
+        elif exp.hypothesis.component == "Workflow":
+            exp = self.workflow_coder.develop(exp)
+        else:
+            raise NotImplementedError(f"Unsupported component in DataScienceRDLoop: {exp.hypothesis.component}")
+
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        if self.trace.all_components_completed():
+            exp = self.runner.develop(prev_out["coding"])
+        else:
+            exp = prev_out["coding"]
+        return exp
+
+    def feedback(self, prev_out: dict[str, Any]):
+        if self.trace.all_components_completed():
+            feedback = self.summarizer.generate_feedback(
+                prev_out["running"], prev_out["direct_exp_gen"].hypothesis, self.trace
+            )
+        else:
+            feedback = HypothesisFeedback(
+                observations="Not all 5 components are completed, skip feedback of DataScienceRDLoop.",
+                hypothesis_evaluation="",
+                new_hypothesis="",
+                reason="",
+                decision=True,
+            )
+        self.trace.hist.append((prev_out["direct_exp_gen"].hypothesis, prev_out["running"], feedback))
+
+
+def main(path=None, step_n=None, competition="bms-molecular-translation"):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+    You can continue running session by
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/data_science/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+    """
+    if competition is not None:
+        DS_RD_SETTING.competition = competition
+
+    if DS_RD_SETTING.competition:
+        if DS_RD_SETTING.scen.endswith("KaggleScen"):
+            download_data(competition=DS_RD_SETTING.competition, settings=DS_RD_SETTING)
+        else:
+            if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists():
+                logger.error(f"Please prepare data for competition {competition} first.")
+                return
+    else:
+        logger.error("Please specify competition name.")
+    if path is None:
+        kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
+    else:
+        kaggle_loop = DataScienceRDLoop.load(path)
+    kaggle_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/kaggle/conf.py

@@ -1,8 +1,7 @@
-from rdagent.components.workflow.conf import BasePropSetting
-from rdagent.core.conf import ExtendedSettingsConfigDict
+from rdagent.core.conf import ExtendedBaseSettings, ExtendedSettingsConfigDict
 
 
-class KaggleBasePropSetting(BasePropSetting):
+class KaggleBasePropSetting(ExtendedBaseSettings):
     model_config = ExtendedSettingsConfigDict(env_prefix="KG_", protected_namespaces=())
 
     # 1) overriding the default
@@ -30,7 +29,7 @@ class KaggleBasePropSetting(BasePropSetting):
     model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
     """Model Runner class"""
 
-    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback"
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10
@@ -45,12 +44,21 @@ class KaggleBasePropSetting(BasePropSetting):
     local_data_path: str = ""
     """Folder storing Kaggle competition data"""
 
+    if_using_mle_data: bool = False
+    auto_submit: bool = False
+    """Automatically upload and submit each experiment result to Kaggle platform"""
+    # Conditionally set the knowledge_base based on the use of graph RAG
+    knowledge_base: str = ""
+    """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""
     if_action_choosing_based_on_UCB: bool = False
     """Enable decision mechanism based on UCB algorithm"""
 
     domain_knowledge_path: str = "/data/userdata/share/kaggle/domain_knowledge"
     """Folder storing domain knowledge files in .case format"""
 
+    knowledge_base_path: str = "kg_graph.pkl"
+    """Advanced version of graph-based RAG"""
+
     rag_path: str = "git_ignore_folder/kaggle_vector_base.pkl"
     """Base version of vector-based RAG"""
 
@@ -60,20 +68,8 @@ class KaggleBasePropSetting(BasePropSetting):
     if_using_graph_rag: bool = False
     """Enable advanced graph-based RAG"""
 
-    # Conditionally set the knowledge_base based on the use of graph RAG
-    knowledge_base: str = ""
-    """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""
-
-    knowledge_base_path: str = "kg_graph.pkl"
-    """Advanced version of graph-based RAG"""
-
-    auto_submit: bool = False
-    """Automatically upload and submit each experiment result to Kaggle platform"""
-
     mini_case: bool = False
     """Enable mini-case study for experiments"""
 
-    if_using_mle_data: bool = False
-
 
 KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/app/kaggle/loop.py

@@ -9,14 +9,13 @@
 from rdagent.core.developer import Developer
 from rdagent.core.exception import FactorEmptyError, ModelEmptyError
 from rdagent.core.proposal import (
+    Experiment2Feedback,
     Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
     HypothesisGen,
 )
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
-from rdagent.log.time import measure_time
 from rdagent.scenarios.kaggle.experiment.scenario import (
     KG_ACTION_FEATURE_ENGINEERING,
     KG_ACTION_FEATURE_PROCESSING,
@@ -28,7 +27,6 @@
 
 
 class KaggleRDLoop(RDLoop):
-    @measure_time
     def __init__(self, PROP_SETTING: BasePropSetting):
         with logger.tag("init"):
             scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
@@ -55,27 +53,31 @@ def __init__(self, PROP_SETTING: BasePropSetting):
             logger.log_object(self.feature_runner, tag="feature runner")
             self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
             logger.log_object(self.model_runner, tag="model runner")
-            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
             logger.log_object(self.summarizer, tag="summarizer")
             self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
             super(RDLoop, self).__init__()
 
-    @measure_time
     def coding(self, prev_out: dict[str, Any]):
         with logger.tag("d"):  # develop
-            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
-                exp = self.feature_coder.develop(prev_out["exp_gen"])
-            elif prev_out["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
-                exp = self.model_feature_selection_coder.develop(prev_out["exp_gen"])
+            if prev_out["direct_exp_gen"]["propose"].action in [
+                KG_ACTION_FEATURE_ENGINEERING,
+                KG_ACTION_FEATURE_PROCESSING,
+            ]:
+                exp = self.feature_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+            elif prev_out["direct_exp_gen"]["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
+                exp = self.model_feature_selection_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
             else:
-                exp = self.model_coder.develop(prev_out["exp_gen"])
+                exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
             logger.log_object(exp.sub_workspace_list, tag="coder result")
         return exp
 
-    @measure_time
     def running(self, prev_out: dict[str, Any]):
         with logger.tag("ef"):  # evaluate and feedback
-            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+            if prev_out["direct_exp_gen"]["propose"].action in [
+                KG_ACTION_FEATURE_ENGINEERING,
+                KG_ACTION_FEATURE_PROCESSING,
+            ]:
                 exp = self.feature_runner.develop(prev_out["coding"])
             else:
                 exp = self.model_runner.develop(prev_out["coding"])
@@ -126,7 +128,7 @@ def main(path=None, step_n=None, competition=None):
     """
     if competition:
         KAGGLE_IMPLEMENT_SETTING.competition = competition
-        download_data(competition=competition, local_path=KAGGLE_IMPLEMENT_SETTING.local_data_path)
+        download_data(competition=competition, settings=KAGGLE_IMPLEMENT_SETTING)
         if KAGGLE_IMPLEMENT_SETTING.if_using_graph_rag:
             KAGGLE_IMPLEMENT_SETTING.knowledge_base = (
                 "rdagent.scenarios.kaggle.knowledge_management.graph.KGKnowledgeGraph"

rdagent/app/qlib_rd_loop/conf.py

@@ -21,7 +21,7 @@ class ModelBasePropSetting(BasePropSetting):
     runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
     """Runner class"""
 
-    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelHypothesisExperiment2Feedback"
+    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10
@@ -47,7 +47,7 @@ class FactorBasePropSetting(BasePropSetting):
     runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
     """Runner class"""
 
-    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorHypothesisExperiment2Feedback"
+    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10

rdagent/app/qlib_rd_loop/factor.py

@@ -10,13 +10,11 @@
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import FactorEmptyError
 from rdagent.log import rdagent_logger as logger
-from rdagent.log.time import measure_time
 
 
 class FactorRDLoop(RDLoop):
     skip_loop_error = (FactorEmptyError,)
 
-    @measure_time
     def running(self, prev_out: dict[str, Any]):
         with logger.tag("ef"):  # evaluate and feedback
             exp = self.runner.develop(prev_out["coding"])