From 8726ab87e69b9e03d2b8835545c5c2b5a7307b91 Mon Sep 17 00:00:00 2001
From: Xisen-Wang <xisen_application@163.com>
Date: Mon, 4 Nov 2024 02:25:20 +0000
Subject: [PATCH] Revising to enable automation of experiments running v1.0

---
 .gitignore                                    |  11 +-
 .../kaggle/automated_evaluation/eval.sh       | 136 ++++++++++++++++++
 2 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100755 rdagent/scenarios/kaggle/automated_evaluation/eval.sh

diff --git a/.gitignore b/.gitignore
index 400cf7d8..7ec4ee96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -151,7 +151,7 @@ reports/
 # git_ignore_folder
 git_ignore_folder/
 
-#cache
+# cache
 *cache*/
 *cache.json
 
@@ -169,4 +169,11 @@ mlruns/
 
 # shell script
 *.out
-*.sh
+
+# Logs
+*.log
+logs/
+log/
+
+# Ignore results directory
+RD-Agent/rdagent/scenarios/kaggle/automated_evaluation/results/
diff --git a/rdagent/scenarios/kaggle/automated_evaluation/eval.sh b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh
new file mode 100755
index 00000000..aaed70ac
--- /dev/null
+++ b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# Comments
+cat << "EOF" > /dev/null
+Experiment Setup Types:
+1. DS-Agent Mini-Case
+2. RD-Agent Basic
+3. RD-Agent Pro
+4. RD-Agent Max
+
+Each setup has specific configurations for:
+- base_model (4o|mini|4o)
+- rag_param (No|Simple|Advanced)
+- if_MAB (True|False)
+- if_feature_selection (True|False)
+- if_hypothesis_proposal (True|False)
+EOF
+
+# Get current time and script directory
+SCRIPT_PATH="$(realpath "$0")"
+SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
+current_time=$(date +"%Y%m%d_%H%M%S")
+export SCRIPT_DIR
+export current_time
+
+# Parse command line arguments
+PARALLEL=1
+CONF_PATH=./
+COMPETITION=""
+SETUP_TYPE=""
+
+while getopts ":sc:k:t:" opt; do
+    case $opt in
+        s)
+        echo "Disable parallel running (run experiments serially)" >&2
+        PARALLEL=0
+        ;;
+        c)
+        echo "Setting conf path $OPTARG" >&2
+        CONF_PATH=$OPTARG
+        ;;
+        k)
+        echo "Setting Kaggle competition $OPTARG" >&2
+        COMPETITION=$OPTARG
+        ;;
+        t)
+        echo "Setting setup type $OPTARG" >&2
+        SETUP_TYPE=$OPTARG
+        ;;
+        \?)
+        echo "Invalid option: -$OPTARG" >&2
+        exit 1
+        ;;
+    esac
+done
+
+# Validate required parameters
+if [ -z "$COMPETITION" ] || [ -z "$SETUP_TYPE" ]; then
+    echo "Error: Competition (-k) and setup type (-t) are required"
+    exit 1
+fi
+
+# Create necessary directories
+mkdir -p "${SCRIPT_DIR}/results/${current_time}"
+mkdir -p "${SCRIPT_DIR}/logs/${current_time}"
+
+# Configure experiment based on setup type
+configure_experiment() {
+    local setup=$1
+    case $setup in
+        "mini-case")
+            echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env"
+            echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env"
+            echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env"
+            echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env"
+            echo "hypothesis_gen=False" >> "${SCRIPT_DIR}/override.env"
+            ;;
+        "basic")
+            echo "if_using_vector_rag=False" > "${SCRIPT_DIR}/override.env"
+            echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env"
+            echo "if_action_choosing_based_on_UCB=False" >> "${SCRIPT_DIR}/override.env"
+            echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env"
+            echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env"
+            ;;
+        "pro")
+            echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env"
+            echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env"
+            echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env"
+            echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env"
+            echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env"
+            ;;
+        "max")
+            echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env"
+            echo "if_using_graph_rag=True" >> "${SCRIPT_DIR}/override.env"
+            echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env"
+            echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env"
+            echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env"
+            ;;
+    esac
+}
+
+# Execute experiment
+run_experiment() {
+    local setup_type=$1
+    local competition=$2
+    
+    configure_experiment "$setup_type"
+    
+    # Run the main experiment loop
+    python -m rdagent.app.kaggle.loop \
+        --competition "$competition" \
+        --setup "$setup_type" \
+        --result_path "${SCRIPT_DIR}/results/${current_time}/result.json" \
+        >> "${SCRIPT_DIR}/logs/${current_time}/experiment.log" 2>&1
+    
+    # Store experiment setup and results
+    cat > "${SCRIPT_DIR}/results/${current_time}/experiment_info.json" << EOF
+{
+    "setup": {
+        "competition": "$competition",
+        "setup_type": "$setup_type",
+        "timestamp": "$current_time"
+    },
+    "results": $(cat "${SCRIPT_DIR}/results/${current_time}/result.json")
+}
+EOF
+}
+
+# Run the experiment
+run_experiment "$SETUP_TYPE" "$COMPETITION"
+
+# Cleanup
+trap 'rm -f "${SCRIPT_DIR}/override.env"' EXIT
+
+echo "Experiment completed. Results are stored in ${SCRIPT_DIR}/results/${current_time}"
+ 
\ No newline at end of file