From 8726ab87e69b9e03d2b8835545c5c2b5a7307b91 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 4 Nov 2024 02:25:20 +0000 Subject: [PATCH 01/21] Revising to enable automation of experiments running v1.0 --- .gitignore | 11 +- .../kaggle/automated_evaluation/eval.sh | 136 ++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100755 rdagent/scenarios/kaggle/automated_evaluation/eval.sh diff --git a/.gitignore b/.gitignore index 400cf7d8..7ec4ee96 100644 --- a/.gitignore +++ b/.gitignore @@ -151,7 +151,7 @@ reports/ # git_ignore_folder git_ignore_folder/ -#cache +# cache *cache*/ *cache.json @@ -169,4 +169,11 @@ mlruns/ # shell script *.out -*.sh + +# Logs +*.log +logs/ +log/ + +# Ignore results directory +RD-Agent/rdagent/scenarios/kaggle/automated_evaluation/results/ diff --git a/rdagent/scenarios/kaggle/automated_evaluation/eval.sh b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh new file mode 100755 index 00000000..aaed70ac --- /dev/null +++ b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Comments +cat << "EOF" > /dev/null +Experiment Setup Types: +1. DS-Agent Mini-Case +2. RD-Agent Basic +3. RD-Agent Pro +4. RD-Agent Max + +Each setup has specific configurations for: +- base_model (4o|mini|4o) +- rag_param (No|Simple|Advanced) +- if_MAB (True|False) +- if_feature_selection (True|False) +- if_hypothesis_proposal (True|False) +EOF + +# Get current time and script directory +SCRIPT_PATH="$(realpath "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +current_time=$(date +"%Y%m%d_%H%M%S") +export SCRIPT_DIR +export current_time + +# Parse command line arguments +PARALLEL=1 +CONF_PATH=./ +COMPETITION="" +SETUP_TYPE="" + +while getopts ":sc:k:t:" opt; do + case $opt in + s) + echo "Disable parallel running (run experiments serially)" >&2 + PARALLEL=0 + ;; + c) + echo "Setting conf path $OPTARG" >&2 + CONF_PATH=$OPTARG + ;; + k) + echo "Setting Kaggle competition $OPTARG" >&2 + COMPETITION=$OPTARG + ;; + t) + echo "Setting setup type $OPTARG" >&2 + SETUP_TYPE=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +# Validate required parameters +if [ -z "$COMPETITION" ] || [ -z "$SETUP_TYPE" ]; then + echo "Error: Competition (-k) and setup type (-t) are required" + exit 1 +fi + +# Create necessary directories +mkdir -p "${SCRIPT_DIR}/results/${current_time}" +mkdir -p "${SCRIPT_DIR}/logs/${current_time}" + +# Configure experiment based on setup type +configure_experiment() { + local setup=$1 + case $setup in + "mini-case") + echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=False" >> "${SCRIPT_DIR}/override.env" + ;; + "basic") + echo "if_using_vector_rag=False" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=False" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" + ;; + "pro") + echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" + ;; + "max") + echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=True" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" + ;; + esac +} + +# Execute experiment +run_experiment() { + local setup_type=$1 + local competition=$2 + + configure_experiment "$setup_type" + + # Run the main experiment loop + python -m rdagent.app.kaggle.loop \ + --competition "$competition" \ + --setup "$setup_type" \ + --result_path "${SCRIPT_DIR}/results/${current_time}/result.json" \ + >> "${SCRIPT_DIR}/logs/${current_time}/experiment.log" 2>&1 + + # Store experiment setup and results + cat > "${SCRIPT_DIR}/results/${current_time}/experiment_info.json" << EOF +{ + "setup": { + "competition": "$competition", + "setup_type": "$setup_type", + "timestamp": "$current_time" + }, + "results": $(cat "${SCRIPT_DIR}/results/${current_time}/result.json") +} +EOF +} + +# Run the experiment +run_experiment "$SETUP_TYPE" "$COMPETITION" + +# Cleanup +trap 'rm -f "${SCRIPT_DIR}/override.env"' EXIT + +echo "Experiment completed. Results are stored in ${SCRIPT_DIR}/results/${current_time}" + \ No newline at end of file From b44bef5ec1546a26acca6ce0c84656d585417df0 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Fri, 15 Nov 2024 15:46:21 +0000 Subject: [PATCH 02/21] Any new updates --- .../results/20241107_051618/experiment_info.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json diff --git a/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json b/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json new file mode 100644 index 00000000..7ceab17e --- /dev/null +++ b/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json @@ -0,0 +1,8 @@ +{ + "setup": { + "competition": "sf-crime", + "setup_type": "mini-case", + "timestamp": "20241107_051618" + }, + "results": +} From c100876008366c87df92ef5841f96f60aac5d1d2 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 4 Nov 2024 02:25:20 +0000 Subject: [PATCH 03/21] Revising to enable automation of experiments running v1.0 --- .gitignore | 11 +- .../kaggle/automated_evaluation/eval.sh | 136 ++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100755 rdagent/scenarios/kaggle/automated_evaluation/eval.sh diff --git a/.gitignore b/.gitignore index 400cf7d8..7ec4ee96 100644 --- a/.gitignore +++ b/.gitignore @@ -151,7 +151,7 @@ reports/ # git_ignore_folder git_ignore_folder/ -#cache +# cache *cache*/ *cache.json @@ -169,4 +169,11 @@ mlruns/ # shell script *.out -*.sh + +# Logs +*.log +logs/ +log/ + +# Ignore results directory +RD-Agent/rdagent/scenarios/kaggle/automated_evaluation/results/ diff --git a/rdagent/scenarios/kaggle/automated_evaluation/eval.sh b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh new file mode 100755 index 00000000..aaed70ac --- /dev/null +++ b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Comments +cat << "EOF" > /dev/null +Experiment Setup Types: +1. DS-Agent Mini-Case +2. RD-Agent Basic +3. RD-Agent Pro +4. RD-Agent Max + +Each setup has specific configurations for: +- base_model (4o|mini|4o) +- rag_param (No|Simple|Advanced) +- if_MAB (True|False) +- if_feature_selection (True|False) +- if_hypothesis_proposal (True|False) +EOF + +# Get current time and script directory +SCRIPT_PATH="$(realpath "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +current_time=$(date +"%Y%m%d_%H%M%S") +export SCRIPT_DIR +export current_time + +# Parse command line arguments +PARALLEL=1 +CONF_PATH=./ +COMPETITION="" +SETUP_TYPE="" + +while getopts ":sc:k:t:" opt; do + case $opt in + s) + echo "Disable parallel running (run experiments serially)" >&2 + PARALLEL=0 + ;; + c) + echo "Setting conf path $OPTARG" >&2 + CONF_PATH=$OPTARG + ;; + k) + echo "Setting Kaggle competition $OPTARG" >&2 + COMPETITION=$OPTARG + ;; + t) + echo "Setting setup type $OPTARG" >&2 + SETUP_TYPE=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +# Validate required parameters +if [ -z "$COMPETITION" ] || [ -z "$SETUP_TYPE" ]; then + echo "Error: Competition (-k) and setup type (-t) are required" + exit 1 +fi + +# Create necessary directories +mkdir -p "${SCRIPT_DIR}/results/${current_time}" +mkdir -p "${SCRIPT_DIR}/logs/${current_time}" + +# Configure experiment based on setup type +configure_experiment() { + local setup=$1 + case $setup in + "mini-case") + echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=False" >> "${SCRIPT_DIR}/override.env" + ;; + "basic") + echo "if_using_vector_rag=False" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=False" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" + ;; + "pro") + echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" + ;; + "max") + echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" + echo "if_using_graph_rag=True" >> "${SCRIPT_DIR}/override.env" + echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" + echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" + echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" + ;; + esac +} + +# Execute experiment +run_experiment() { + local setup_type=$1 + local competition=$2 + + configure_experiment "$setup_type" + + # Run the main experiment loop + python -m rdagent.app.kaggle.loop \ + --competition "$competition" \ + --setup "$setup_type" \ + --result_path "${SCRIPT_DIR}/results/${current_time}/result.json" \ + >> "${SCRIPT_DIR}/logs/${current_time}/experiment.log" 2>&1 + + # Store experiment setup and results + cat > "${SCRIPT_DIR}/results/${current_time}/experiment_info.json" << EOF +{ + "setup": { + "competition": "$competition", + "setup_type": "$setup_type", + "timestamp": "$current_time" + }, + "results": $(cat "${SCRIPT_DIR}/results/${current_time}/result.json") +} +EOF +} + +# Run the experiment +run_experiment "$SETUP_TYPE" "$COMPETITION" + +# Cleanup +trap 'rm -f "${SCRIPT_DIR}/override.env"' EXIT + +echo "Experiment completed. Results are stored in ${SCRIPT_DIR}/results/${current_time}" + \ No newline at end of file From 18370d4c228db60897d6b49ac06f5291f81b1841 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Fri, 15 Nov 2024 15:46:21 +0000 Subject: [PATCH 04/21] Any new updates --- .../results/20241107_051618/experiment_info.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json diff --git a/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json b/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json new file mode 100644 index 00000000..7ceab17e --- /dev/null +++ b/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json @@ -0,0 +1,8 @@ +{ + "setup": { + "competition": "sf-crime", + "setup_type": "mini-case", + "timestamp": "20241107_051618" + }, + "results": +} From 21a99d27de948a9a3dc527c24dd73862c3a74c99 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 15 Nov 2024 16:15:58 +0000 Subject: [PATCH 05/21] Add template --- scripts/exp/ablation/README.md | 5 ++++ scripts/exp/ablation/env/full.env | 1 + scripts/exp/tools/README.md | 3 ++ scripts/exp/tools/collect.py | 0 scripts/exp/tools/run_envs.sh | 46 +++++++++++++++++++++++++++++++ 5 files changed, 55 insertions(+) create mode 100644 scripts/exp/ablation/README.md create mode 100644 scripts/exp/ablation/env/full.env create mode 100644 scripts/exp/tools/README.md create mode 100644 scripts/exp/tools/collect.py create mode 100644 scripts/exp/tools/run_envs.sh diff --git a/scripts/exp/ablation/README.md b/scripts/exp/ablation/README.md new file mode 100644 index 00000000..2b4df02d --- /dev/null +++ b/scripts/exp/ablation/README.md @@ -0,0 +1,5 @@ +# Introduction + +| name | .env | desc | +| -- | -- | -- | +| full | full.env | enable all features | diff --git a/scripts/exp/ablation/env/full.env b/scripts/exp/ablation/env/full.env new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/exp/ablation/env/full.env @@ -0,0 +1 @@ + diff --git a/scripts/exp/tools/README.md b/scripts/exp/tools/README.md new file mode 100644 index 00000000..659fbf8b --- /dev/null +++ b/scripts/exp/tools/README.md @@ -0,0 +1,3 @@ +The tools in the directory contains following generalfeatures +- collecting envs and run each +- collect results and generate summary diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/exp/tools/run_envs.sh b/scripts/exp/tools/run_envs.sh new file mode 100644 index 00000000..2653b576 --- /dev/null +++ b/scripts/exp/tools/run_envs.sh @@ -0,0 +1,46 @@ +#!/bin/sh +cat << "EOF" > /dev/null +Given a directory with *.env files. Run each one. + +usage for example: + + 1) directly run command without extra shared envs + ./run_envs.sh -d -j -- + + 2) load shared envs `.env` before running command with different envs. + dotenv run -- ./run_envs.sh -d -j -- + +EOF + +# Function to display usage +usage() { + echo "Usage: $0 -d -j -- " + exit 1 +} + +# Parse command line arguments +while getopts "d:j:" opt; do + case $opt in + d) DIR=$OPTARG ;; + j) JOBS=$OPTARG ;; + *) usage ;; + esac +done + +# Shift to get the command +shift $((OPTIND -1)) + +# Check if directory and jobs are set +if [ -z "$DIR" ] || [ -z "$OBS" ] || [ $# -eq 0 ]; then + usage +fi + +COMMAND="$@" + +# Export and run each .env file in parallel +find "$DIR" -name "*.env" | xargs -n 1 -P "$JOBS" -I {} sh -c " + set -a + . {} + set +a + $COMMAND +" From 86ae0b20d01dcaca88058c9e4c0326b8301074ff Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 20 Nov 2024 08:23:57 +0000 Subject: [PATCH 06/21] Stoping tracking additional env --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7ec4ee96..35ce7e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.huaxia_env # Spyder project settings .spyderproject From 66ffd6d42bc7d78f499401813b2f8d132155ef55 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 20 Nov 2024 09:13:42 +0000 Subject: [PATCH 07/21] Uploading relevant envs --- scripts/exp/ablation/README.md | 4 ++++ scripts/exp/ablation/env/basic.env | 5 +++++ scripts/exp/ablation/env/max.env | 5 +++++ scripts/exp/ablation/env/mini-case.env | 5 +++++ scripts/exp/ablation/env/pro.env | 5 +++++ 5 files changed, 24 insertions(+) create mode 100644 scripts/exp/ablation/env/basic.env create mode 100644 scripts/exp/ablation/env/max.env create mode 100644 scripts/exp/ablation/env/mini-case.env create mode 100644 scripts/exp/ablation/env/pro.env diff --git a/scripts/exp/ablation/README.md b/scripts/exp/ablation/README.md index 2b4df02d..72b8c512 100644 --- a/scripts/exp/ablation/README.md +++ b/scripts/exp/ablation/README.md @@ -3,3 +3,7 @@ | name | .env | desc | | -- | -- | -- | | full | full.env | enable all features | +| minicase | minicase.env | enable minicase | + + + diff --git a/scripts/exp/ablation/env/basic.env b/scripts/exp/ablation/env/basic.env new file mode 100644 index 00000000..480a563c --- /dev/null +++ b/scripts/exp/ablation/env/basic.env @@ -0,0 +1,5 @@ +if_using_vector_rag=False +if_using_graph_rag=False +if_action_choosing_based_on_UCB=False +model_feature_selection_coder=True +hypothesis_gen=True \ No newline at end of file diff --git a/scripts/exp/ablation/env/max.env b/scripts/exp/ablation/env/max.env new file mode 100644 index 00000000..a4f8f5f1 --- /dev/null +++ b/scripts/exp/ablation/env/max.env @@ -0,0 +1,5 @@ +if_using_vector_rag=True +if_using_graph_rag=True +if_action_choosing_based_on_UCB=True +model_feature_selection_coder=True +hypothesis_gen=True \ No newline at end of file diff --git a/scripts/exp/ablation/env/mini-case.env b/scripts/exp/ablation/env/mini-case.env new file mode 100644 index 00000000..3af468c7 --- /dev/null +++ b/scripts/exp/ablation/env/mini-case.env @@ -0,0 +1,5 @@ +if_using_vector_rag=True +if_using_graph_rag=False +if_action_choosing_based_on_UCB=True +model_feature_selection_coder=True +hypothesis_gen=False \ No newline at end of file diff --git a/scripts/exp/ablation/env/pro.env b/scripts/exp/ablation/env/pro.env new file mode 100644 index 00000000..ac9e152c --- /dev/null +++ b/scripts/exp/ablation/env/pro.env @@ -0,0 +1,5 @@ +if_using_vector_rag=True +if_using_graph_rag=False +if_action_choosing_based_on_UCB=True +model_feature_selection_coder=True +hypothesis_gen=True \ No newline at end of file From 0ef80a558a5a224fd7e4246bac04437f5e66d42b Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 20 Nov 2024 14:19:39 +0000 Subject: [PATCH 08/21] Adding tests --- scripts/exp/tools/collect.py | 28 ++++++++++++++++++++++++++++ scripts/exp/tools/run_envs.sh | 7 ++++++- scripts/exp/tools/test_system.sh | 21 +++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 scripts/exp/tools/test_system.sh diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index e69de29b..631515b9 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -0,0 +1,28 @@ +import os +import json + +def collect_results(dir_path) -> list[dict]: + summary = [] + for root, _, filies in os.walk(dir_path): + for file in filies: + if file.endswith(".json"): + with open(os.path.join(root, file), "r") as f: + data = json.load(f) + summary.append(data) + return summary + +def generate_summary(results, output_path): + # First analyze the results and generate a summary + # For each experiment, we find the best result, the metric, and result trajectory + #TODO: Implement this + + # Then write the summary to the output path + with open(output_path, "w") as f: + json.dump(results, f, indent = 4) + +if __name__ == "__main__": + result_dir = os.path.join(os.getenv("EXP_DIR"), "results") + results = collect_results(result_dir) + generate_summary(results, os.path.join(result_dir, "summary.json")) + print("Summary generated successfully at ", os.path.join(result_dir, "summary.json")) + diff --git a/scripts/exp/tools/run_envs.sh b/scripts/exp/tools/run_envs.sh index 2653b576..9377b6e2 100644 --- a/scripts/exp/tools/run_envs.sh +++ b/scripts/exp/tools/run_envs.sh @@ -31,12 +31,16 @@ done shift $((OPTIND -1)) # Check if directory and jobs are set -if [ -z "$DIR" ] || [ -z "$OBS" ] || [ $# -eq 0 ]; then +if [ -z "$DIR" ] || [ -z "$JOBS" ] || [ $# -eq 0 ]; then usage fi COMMAND="$@" +# Before running commands +echo "Running experiments with following env files:" +find "$DIR" -name "*.env" -exec echo "{}" \; + # Export and run each .env file in parallel find "$DIR" -name "*.env" | xargs -n 1 -P "$JOBS" -I {} sh -c " set -a @@ -44,3 +48,4 @@ find "$DIR" -name "*.env" | xargs -n 1 -P "$JOBS" -I {} sh -c " set +a $COMMAND " + diff --git a/scripts/exp/tools/test_system.sh b/scripts/exp/tools/test_system.sh new file mode 100644 index 00000000..8c136edc --- /dev/null +++ b/scripts/exp/tools/test_system.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Test directory setup +TEST_DIR="test_run" +mkdir -p "$TEST_DIR/results" + +# Test 1: Environment loading +echo "Testing environment loading..." +./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 1 -- env | grep "if_using" + +# Test 2: Parallel execution +echo "Testing parallel execution..." +./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 4 -- \ + echo "Processing env with RAG setting: $if_using_vector_rag" + +# Test 3: Result collection +echo "Testing result collection..." +EXP_DIR="$TEST_DIR" python scripts/exp/tools/collect.py + +# Cleanup +rm -rf "$TEST_DIR" \ No newline at end of file From 907d9805d848f0b398baaa70be2989e07a814898 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 20 Nov 2024 14:53:04 +0000 Subject: [PATCH 09/21] Updating --- scripts/exp/tools/collect.py | 60 ++++++++++++++++++++++++++------ scripts/exp/tools/test_system.sh | 16 ++++++--- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index 631515b9..ec910537 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -1,24 +1,64 @@ import os import json +from pathlib import Path +from datetime import datetime def collect_results(dir_path) -> list[dict]: summary = [] - for root, _, filies in os.walk(dir_path): - for file in filies: - if file.endswith(".json"): + for root, _, files in os.walk(dir_path): + for file in files: + if file.endswith("_result.json"): + config_name = file.replace("_result.json", "") with open(os.path.join(root, file), "r") as f: data = json.load(f) - summary.append(data) + # Extract both CV and Kaggle submission results + summary.append({ + "config": config_name, + "cv_results": data.get("cv_score", None), + "kaggle_score": data.get("kaggle_score", None), + "trace": data.get("trace", {}) + }) return summary def generate_summary(results, output_path): - # First analyze the results and generate a summary - # For each experiment, we find the best result, the metric, and result trajectory - #TODO: Implement this - - # Then write the summary to the output path + summary = { + "configs": {}, + "best_cv_result": {"config": None, "score": None}, + "best_kaggle_result": {"config": None, "score": None}, + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S") + } + + for result in results: + config = result["config"] + metrics = { + "cv_score": result["cv_results"], + "kaggle_score": result["kaggle_score"], + "iterations": len(result["trace"].get("steps", [])), + "best_model": result["trace"].get("best_model") + } + + summary["configs"][config] = metrics + + # Update best CV result + if (metrics["cv_score"] is not None and + (summary["best_cv_result"]["score"] is None or + metrics["cv_score"] > summary["best_cv_result"]["score"])): + summary["best_cv_result"].update({ + "config": config, + "score": metrics["cv_score"] + }) + + # Update best Kaggle result + if (metrics["kaggle_score"] is not None and + (summary["best_kaggle_result"]["score"] is None or + metrics["kaggle_score"] > summary["best_kaggle_result"]["score"])): + summary["best_kaggle_result"].update({ + "config": config, + "score": metrics["kaggle_score"] + }) + with open(output_path, "w") as f: - json.dump(results, f, indent = 4) + json.dump(summary, f, indent=4) if __name__ == "__main__": result_dir = os.path.join(os.getenv("EXP_DIR"), "results") diff --git a/scripts/exp/tools/test_system.sh b/scripts/exp/tools/test_system.sh index 8c136edc..95bfcb9c 100644 --- a/scripts/exp/tools/test_system.sh +++ b/scripts/exp/tools/test_system.sh @@ -3,19 +3,25 @@ # Test directory setup TEST_DIR="test_run" mkdir -p "$TEST_DIR/results" +mkdir -p "$TEST_DIR/logs" -# Test 1: Environment loading +# Test 1: Environment loading verification echo "Testing environment loading..." ./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 1 -- env | grep "if_using" -# Test 2: Parallel execution -echo "Testing parallel execution..." +# Test 2: Run actual experiments +echo "Running experiments with different configurations..." ./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 4 -- \ - echo "Processing env with RAG setting: $if_using_vector_rag" + python -m rdagent.app.kaggle.loop \ + --competition "titanic" \ + --result_path "${TEST_DIR}/results/$(basename {} .env)_result.json" # Test 3: Result collection -echo "Testing result collection..." +echo "Collecting and analyzing results..." EXP_DIR="$TEST_DIR" python scripts/exp/tools/collect.py +# Display results location +echo "Test results available at: $TEST_DIR" + # Cleanup rm -rf "$TEST_DIR" \ No newline at end of file From 51388d1cee286e054b226e5dc6ec83f3b127902f Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Sat, 23 Nov 2024 03:07:59 +0000 Subject: [PATCH 10/21] Updated collect.py to extract result from trace --- scripts/exp/tools/collect.py | 55 ++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index ec910537..3ccc4f06 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -2,6 +2,7 @@ import json from pathlib import Path from datetime import datetime +from rdagent.log.storage import FileStorage def collect_results(dir_path) -> list[dict]: summary = [] @@ -9,52 +10,46 @@ def collect_results(dir_path) -> list[dict]: for file in files: if file.endswith("_result.json"): config_name = file.replace("_result.json", "") - with open(os.path.join(root, file), "r") as f: - data = json.load(f) - # Extract both CV and Kaggle submission results - summary.append({ - "config": config_name, - "cv_results": data.get("cv_score", None), - "kaggle_score": data.get("kaggle_score", None), - "trace": data.get("trace", {}) - }) + log_storage = FileStorage(Path(root)) + + score = None + # Extract score from trace using the same approach as UI + for msg in log_storage.iter_msg(): + if "runner result" in msg.tag: + if msg.content.result is not None: + score = msg.content.result + break + + summary.append({ + "config": config_name, + "score": score, + "workspace": str(root) + }) return summary def generate_summary(results, output_path): summary = { "configs": {}, - "best_cv_result": {"config": None, "score": None}, - "best_kaggle_result": {"config": None, "score": None}, + "best_result": {"config": None, "score": None}, "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S") } for result in results: config = result["config"] metrics = { - "cv_score": result["cv_results"], - "kaggle_score": result["kaggle_score"], - "iterations": len(result["trace"].get("steps", [])), - "best_model": result["trace"].get("best_model") + "score": result["score"], + "workspace": result["workspace"] } summary["configs"][config] = metrics - # Update best CV result - if (metrics["cv_score"] is not None and - (summary["best_cv_result"]["score"] is None or - metrics["cv_score"] > summary["best_cv_result"]["score"])): - summary["best_cv_result"].update({ + # Update best result + if (result["score"] is not None and + (summary["best_result"]["score"] is None or + result["score"] > summary["best_result"]["score"])): + summary["best_result"].update({ "config": config, - "score": metrics["cv_score"] - }) - - # Update best Kaggle result - if (metrics["kaggle_score"] is not None and - (summary["best_kaggle_result"]["score"] is None or - metrics["kaggle_score"] > summary["best_kaggle_result"]["score"])): - summary["best_kaggle_result"].update({ - "config": config, - "score": metrics["kaggle_score"] + "score": result["score"] }) with open(output_path, "w") as f: From af6220e93575c7d54db951b54530844b40498957 Mon Sep 17 00:00:00 2001 From: Xisen Wang <118058822+xisen-w@users.noreply.github.com> Date: Sat, 23 Nov 2024 03:09:33 +0000 Subject: [PATCH 11/21] Update .gitignore to remove the unecessary ones --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 35ce7e0a..7ec4ee96 100644 --- a/.gitignore +++ b/.gitignore @@ -117,7 +117,6 @@ venv/ ENV/ env.bak/ venv.bak/ -.huaxia_env # Spyder project settings .spyderproject From 54c3c6d1d205696a9348d370d56fff63423743c0 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Sat, 23 Nov 2024 03:17:34 +0000 Subject: [PATCH 12/21] "Remove unnecessary files" --- .../kaggle/automated_evaluation/eval.sh | 136 ------------------ .../20241107_051618/experiment_info.json | 8 -- scripts/exp/tools/collect.py | 55 ++++--- 3 files changed, 25 insertions(+), 174 deletions(-) delete mode 100755 rdagent/scenarios/kaggle/automated_evaluation/eval.sh delete mode 100644 rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json diff --git a/rdagent/scenarios/kaggle/automated_evaluation/eval.sh b/rdagent/scenarios/kaggle/automated_evaluation/eval.sh deleted file mode 100755 index aaed70ac..00000000 --- a/rdagent/scenarios/kaggle/automated_evaluation/eval.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - -# Comments -cat << "EOF" > /dev/null -Experiment Setup Types: -1. DS-Agent Mini-Case -2. RD-Agent Basic -3. RD-Agent Pro -4. RD-Agent Max - -Each setup has specific configurations for: -- base_model (4o|mini|4o) -- rag_param (No|Simple|Advanced) -- if_MAB (True|False) -- if_feature_selection (True|False) -- if_hypothesis_proposal (True|False) -EOF - -# Get current time and script directory -SCRIPT_PATH="$(realpath "$0")" -SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" -current_time=$(date +"%Y%m%d_%H%M%S") -export SCRIPT_DIR -export current_time - -# Parse command line arguments -PARALLEL=1 -CONF_PATH=./ -COMPETITION="" -SETUP_TYPE="" - -while getopts ":sc:k:t:" opt; do - case $opt in - s) - echo "Disable parallel running (run experiments serially)" >&2 - PARALLEL=0 - ;; - c) - echo "Setting conf path $OPTARG" >&2 - CONF_PATH=$OPTARG - ;; - k) - echo "Setting Kaggle competition $OPTARG" >&2 - COMPETITION=$OPTARG - ;; - t) - echo "Setting setup type $OPTARG" >&2 - SETUP_TYPE=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - esac -done - -# Validate required parameters -if [ -z "$COMPETITION" ] || [ -z "$SETUP_TYPE" ]; then - echo "Error: Competition (-k) and setup type (-t) are required" - exit 1 -fi - -# Create necessary directories -mkdir -p "${SCRIPT_DIR}/results/${current_time}" -mkdir -p "${SCRIPT_DIR}/logs/${current_time}" - -# Configure experiment based on setup type -configure_experiment() { - local setup=$1 - case $setup in - "mini-case") - echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" - echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" - echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" - echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" - echo "hypothesis_gen=False" >> "${SCRIPT_DIR}/override.env" - ;; - "basic") - echo "if_using_vector_rag=False" > "${SCRIPT_DIR}/override.env" - echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" - echo "if_action_choosing_based_on_UCB=False" >> "${SCRIPT_DIR}/override.env" - echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" - echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" - ;; - "pro") - echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" - echo "if_using_graph_rag=False" >> "${SCRIPT_DIR}/override.env" - echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" - echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" - echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" - ;; - "max") - echo "if_using_vector_rag=True" > "${SCRIPT_DIR}/override.env" - echo "if_using_graph_rag=True" >> "${SCRIPT_DIR}/override.env" - echo "if_action_choosing_based_on_UCB=True" >> "${SCRIPT_DIR}/override.env" - echo "model_feature_selection_coder=True" >> "${SCRIPT_DIR}/override.env" - echo "hypothesis_gen=True" >> "${SCRIPT_DIR}/override.env" - ;; - esac -} - -# Execute experiment -run_experiment() { - local setup_type=$1 - local competition=$2 - - configure_experiment "$setup_type" - - # Run the main experiment loop - python -m rdagent.app.kaggle.loop \ - --competition "$competition" \ - --setup "$setup_type" \ - --result_path "${SCRIPT_DIR}/results/${current_time}/result.json" \ - >> "${SCRIPT_DIR}/logs/${current_time}/experiment.log" 2>&1 - - # Store experiment setup and results - cat > "${SCRIPT_DIR}/results/${current_time}/experiment_info.json" << EOF -{ - "setup": { - "competition": "$competition", - "setup_type": "$setup_type", - "timestamp": "$current_time" - }, - "results": $(cat "${SCRIPT_DIR}/results/${current_time}/result.json") -} -EOF -} - -# Run the experiment -run_experiment "$SETUP_TYPE" "$COMPETITION" - -# Cleanup -trap 'rm -f "${SCRIPT_DIR}/override.env"' EXIT - -echo "Experiment completed. Results are stored in ${SCRIPT_DIR}/results/${current_time}" - \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json b/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json deleted file mode 100644 index 7ceab17e..00000000 --- a/rdagent/scenarios/kaggle/automated_evaluation/results/20241107_051618/experiment_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "setup": { - "competition": "sf-crime", - "setup_type": "mini-case", - "timestamp": "20241107_051618" - }, - "results": -} diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index ec910537..3ccc4f06 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -2,6 +2,7 @@ import json from pathlib import Path from datetime import datetime +from rdagent.log.storage import FileStorage def collect_results(dir_path) -> list[dict]: summary = [] @@ -9,52 +10,46 @@ def collect_results(dir_path) -> list[dict]: for file in files: if file.endswith("_result.json"): config_name = file.replace("_result.json", "") - with open(os.path.join(root, file), "r") as f: - data = json.load(f) - # Extract both CV and Kaggle submission results - summary.append({ - "config": config_name, - "cv_results": data.get("cv_score", None), - "kaggle_score": data.get("kaggle_score", None), - "trace": data.get("trace", {}) - }) + log_storage = FileStorage(Path(root)) + + score = None + # Extract score from trace using the same approach as UI + for msg in log_storage.iter_msg(): + if "runner result" in msg.tag: + if msg.content.result is not None: + score = msg.content.result + break + + summary.append({ + "config": config_name, + "score": score, + "workspace": str(root) + }) return summary def generate_summary(results, output_path): summary = { "configs": {}, - "best_cv_result": {"config": None, "score": None}, - "best_kaggle_result": {"config": None, "score": None}, + "best_result": {"config": None, "score": None}, "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S") } for result in results: config = result["config"] metrics = { - "cv_score": result["cv_results"], - "kaggle_score": result["kaggle_score"], - "iterations": len(result["trace"].get("steps", [])), - "best_model": result["trace"].get("best_model") + "score": result["score"], + "workspace": result["workspace"] } summary["configs"][config] = metrics - # Update best CV result - if (metrics["cv_score"] is not None and - (summary["best_cv_result"]["score"] is None or - metrics["cv_score"] > summary["best_cv_result"]["score"])): - summary["best_cv_result"].update({ + # Update best result + if (result["score"] is not None and + (summary["best_result"]["score"] is None or + result["score"] > summary["best_result"]["score"])): + summary["best_result"].update({ "config": config, - "score": metrics["cv_score"] - }) - - # Update best Kaggle result - if (metrics["kaggle_score"] is not None and - (summary["best_kaggle_result"]["score"] is None or - metrics["kaggle_score"] > summary["best_kaggle_result"]["score"])): - summary["best_kaggle_result"].update({ - "config": config, - "score": metrics["kaggle_score"] + "score": result["score"] }) with open(output_path, "w") as f: From 38bb9e6df98c554196cd901f21d0160c71a1fd36 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 25 Nov 2024 11:11:40 +0000 Subject: [PATCH 13/21] Updated to enable automatic collection of experiment result information --- scripts/exp/tools/collect.py | 84 +++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index 3ccc4f06..4f162e3d 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -3,61 +3,65 @@ from pathlib import Path from datetime import datetime from rdagent.log.storage import FileStorage +from rdagent.scenarios.kaggle.kaggle_crawler import ( + leaderboard_scores, +) -def collect_results(dir_path) -> list[dict]: +def collect_results(log_path) -> list[dict]: summary = [] - for root, _, files in os.walk(dir_path): - for file in files: - if file.endswith("_result.json"): - config_name = file.replace("_result.json", "") - log_storage = FileStorage(Path(root)) - - score = None - # Extract score from trace using the same approach as UI - for msg in log_storage.iter_msg(): - if "runner result" in msg.tag: - if msg.content.result is not None: - score = msg.content.result - break - + log_storage = FileStorage(Path(log_path)) + evaluation_metric_direction = None + # Extract score from trace using the same approach as UI + for msg in log_storage.iter_msg(): + if "scenario" in msg.tag: + competition_name = msg.content.competition # Find the competition name + leaderboard = leaderboard_scores(competition_name) + evaluation_metric_direction = float(leaderboard[0]) > float(leaderboard[-1]) + + if "runner result" in msg.tag: + if msg.content.result is not None: + score = msg.content.result summary.append({ - "config": config_name, + "competition_name": competition_name, "score": score, - "workspace": str(root) + "workspace": msg.content.experiment_workspace.workspace_path, + "evaluation_metric_direction": evaluation_metric_direction }) return summary def generate_summary(results, output_path): summary = { - "configs": {}, - "best_result": {"config": None, "score": None}, - "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S") + "configs": {}, #TODO: add config? + "best_result": {"competition_name": None, "score": None}, + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + #Add other metrics that we want to track in the future (eg. is there successive increase?) } - for result in results: - config = result["config"] - metrics = { - "score": result["score"], - "workspace": result["workspace"] - } - - summary["configs"][config] = metrics - # Update best result - if (result["score"] is not None and - (summary["best_result"]["score"] is None or - result["score"] > summary["best_result"]["score"])): - summary["best_result"].update({ - "config": config, - "score": result["score"] - }) + # If the evaluation metric is higher, it is better + if result["evaluation_metric_direction"]: + if (result["score"] is not None and + (summary["best_result"]["score"] is None or + result["score"] > summary["best_result"]["score"])): + summary["best_result"].update({ + "score": result["score"], + "competition_name": result["competition_name"] + }) + else: + if (result["score"] is not None and + (summary["best_result"]["score"] is None or + result["score"] < summary["best_result"]["score"])): + summary["best_result"].update({ + "score": result["score"], + "competition_name": result["competition_name"] + }) with open(output_path, "w") as f: json.dump(summary, f, indent=4) if __name__ == "__main__": - result_dir = os.path.join(os.getenv("EXP_DIR"), "results") - results = collect_results(result_dir) - generate_summary(results, os.path.join(result_dir, "summary.json")) - print("Summary generated successfully at ", os.path.join(result_dir, "summary.json")) + sample_result_dir = Path("/home/bowen/workspace/RD-Agent/log/MAY2022_5") + results = collect_results(sample_result_dir ) + generate_summary(results, os.path.join(sample_result_dir, "summary.json")) + print("Summary generated successfully at ", os.path.join(sample_result_dir, "summary.json")) From 10b0053a18448a40fb2348f846b6c8e560de7bff Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 25 Nov 2024 11:45:14 +0000 Subject: [PATCH 14/21] Updating the env files & Upading test_system file --- scripts/exp/ablation/env/basic.env | 10 ++++----- scripts/exp/ablation/env/full.env | 1 - scripts/exp/ablation/env/max.env | 10 ++++----- scripts/exp/ablation/env/mini-case.env | 10 ++++----- scripts/exp/ablation/env/pro.env | 10 ++++----- scripts/exp/tools/collect.py | 26 +++++++++++++++++++----- scripts/exp/tools/run_envs.sh | 0 scripts/exp/tools/test_system.sh | 28 +++++++++----------------- 8 files changed, 51 insertions(+), 44 deletions(-) delete mode 100644 scripts/exp/ablation/env/full.env mode change 100644 => 100755 scripts/exp/tools/run_envs.sh mode change 100644 => 100755 scripts/exp/tools/test_system.sh diff --git a/scripts/exp/ablation/env/basic.env b/scripts/exp/ablation/env/basic.env index 480a563c..256e370e 100644 --- a/scripts/exp/ablation/env/basic.env +++ b/scripts/exp/ablation/env/basic.env @@ -1,5 +1,5 @@ -if_using_vector_rag=False -if_using_graph_rag=False -if_action_choosing_based_on_UCB=False -model_feature_selection_coder=True -hypothesis_gen=True \ No newline at end of file +KG_IF_USING_VECTOR_RAG=False +KG_IF_USING_GRAPH_RAG=False +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=False +KG_MODEL_FEATURE_SELECTION_CODER=True +KG_HYPOTHESIS_GEN=True \ No newline at end of file diff --git a/scripts/exp/ablation/env/full.env b/scripts/exp/ablation/env/full.env deleted file mode 100644 index 8b137891..00000000 --- a/scripts/exp/ablation/env/full.env +++ /dev/null @@ -1 +0,0 @@ - diff --git a/scripts/exp/ablation/env/max.env b/scripts/exp/ablation/env/max.env index a4f8f5f1..40c67404 100644 --- a/scripts/exp/ablation/env/max.env +++ b/scripts/exp/ablation/env/max.env @@ -1,5 +1,5 @@ -if_using_vector_rag=True -if_using_graph_rag=True -if_action_choosing_based_on_UCB=True -model_feature_selection_coder=True -hypothesis_gen=True \ No newline at end of file +KG_IF_USING_VECTOR_RAG=True +KG_IF_USING_GRAPH_RAG=True +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True +KG_MODEL_FEATURE_SELECTION_CODER=True +KG_HYPOTHESIS_GEN=True \ No newline at end of file diff --git a/scripts/exp/ablation/env/mini-case.env b/scripts/exp/ablation/env/mini-case.env index 3af468c7..87b0e716 100644 --- a/scripts/exp/ablation/env/mini-case.env +++ b/scripts/exp/ablation/env/mini-case.env @@ -1,5 +1,5 @@ -if_using_vector_rag=True -if_using_graph_rag=False -if_action_choosing_based_on_UCB=True -model_feature_selection_coder=True -hypothesis_gen=False \ No newline at end of file +KG_IF_USING_VECTOR_RAG=True +KG_IF_USING_GRAPH_RAG=False +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True +KG_MODEL_FEATURE_SELECTION_CODER=True +KG_HYPOTHESIS_GEN=False \ No newline at end of file diff --git a/scripts/exp/ablation/env/pro.env b/scripts/exp/ablation/env/pro.env index ac9e152c..2ba8cc0f 100644 --- a/scripts/exp/ablation/env/pro.env +++ b/scripts/exp/ablation/env/pro.env @@ -1,5 +1,5 @@ -if_using_vector_rag=True -if_using_graph_rag=False -if_action_choosing_based_on_UCB=True -model_feature_selection_coder=True -hypothesis_gen=True \ No newline at end of file +KG_IF_USING_VECTOR_RAG=True +KG_IF_USING_GRAPH_RAG=False +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True +KG_MODEL_FEATURE_SELECTION_CODER=True +KG_HYPOTHESIS_GEN=True \ No newline at end of file diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index 4f162e3d..8ab2d7a2 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -1,5 +1,6 @@ import os import json +import argparse from pathlib import Path from datetime import datetime from rdagent.log.storage import FileStorage @@ -56,12 +57,27 @@ def generate_summary(results, output_path): "competition_name": result["competition_name"] }) - with open(output_path, "w") as f: + with open(output_path, "w") as f: json.dump(summary, f, indent=4) +def parse_args(): + parser = argparse.ArgumentParser(description='Collect and summarize experiment results') + parser.add_argument('--log_path', type=str, required=True, + help='Path to the log directory containing experiment results') + parser.add_argument('--output_name', type=str, default='summary.json', + help='Name of the output summary file (default: summary.json)') + return parser.parse_args() + if __name__ == "__main__": - sample_result_dir = Path("/home/bowen/workspace/RD-Agent/log/MAY2022_5") - results = collect_results(sample_result_dir ) - generate_summary(results, os.path.join(sample_result_dir, "summary.json")) - print("Summary generated successfully at ", os.path.join(sample_result_dir, "summary.json")) + args = parse_args() + log_path = Path(args.log_path) + + # Verify the log path exists + if not log_path.exists(): + raise FileNotFoundError(f"Log path does not exist: {log_path}") + + results = collect_results(log_path) + output_path = log_path / args.output_name + generate_summary(results, output_path) + print("Summary generated successfully at", output_path) diff --git a/scripts/exp/tools/run_envs.sh b/scripts/exp/tools/run_envs.sh old mode 100644 new mode 100755 diff --git a/scripts/exp/tools/test_system.sh b/scripts/exp/tools/test_system.sh old mode 100644 new mode 100755 index 95bfcb9c..4b417cd5 --- a/scripts/exp/tools/test_system.sh +++ b/scripts/exp/tools/test_system.sh @@ -5,23 +5,15 @@ TEST_DIR="test_run" mkdir -p "$TEST_DIR/results" mkdir -p "$TEST_DIR/logs" -# Test 1: Environment loading verification -echo "Testing environment loading..." -./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 1 -- env | grep "if_using" +# Define paths +ENV_DIR="/home/v-xisenwang/RD-Agent/scripts/exp/ablation/env" +PYTHON_SCRIPT="/home/v-xisenwang/RD-Agent/rdagent/app/kaggle/loop.py" -# Test 2: Run actual experiments -echo "Running experiments with different configurations..." -./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 4 -- \ - python -m rdagent.app.kaggle.loop \ - --competition "titanic" \ - --result_path "${TEST_DIR}/results/$(basename {} .env)_result.json" +# Run the experiment +echo "Running experiments..." +dotenv run -- ./scripts/exp/tools/run_envs.sh -d "$ENV_DIR" -j 4 -- \ + python "$PYTHON_SCRIPT" \ + --competition "spaceship-titanic" \ -# Test 3: Result collection -echo "Collecting and analyzing results..." -EXP_DIR="$TEST_DIR" python scripts/exp/tools/collect.py - -# Display results location -echo "Test results available at: $TEST_DIR" - -# Cleanup -rm -rf "$TEST_DIR" \ No newline at end of file +# Cleanup (optional - comment out if you want to keep results) +# rm -rf "$TEST_DIR" \ No newline at end of file From 238f49269438018d7409c5182d384b175e49750c Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 25 Nov 2024 12:06:42 +0000 Subject: [PATCH 15/21] Updated relevant env for better testing --- scripts/exp/ablation/env/basic.env | 4 +- scripts/exp/ablation/env/max.env | 4 +- scripts/exp/ablation/env/mini-case.env | 2 - scripts/exp/ablation/env/pro.env | 2 - scripts/exp/tools/README.md | 141 ++++++++++++++++++++++++- 5 files changed, 140 insertions(+), 13 deletions(-) diff --git a/scripts/exp/ablation/env/basic.env b/scripts/exp/ablation/env/basic.env index 256e370e..60557e2b 100644 --- a/scripts/exp/ablation/env/basic.env +++ b/scripts/exp/ablation/env/basic.env @@ -1,5 +1,3 @@ KG_IF_USING_VECTOR_RAG=False KG_IF_USING_GRAPH_RAG=False -KG_IF_ACTION_CHOOSING_BASED_ON_UCB=False -KG_MODEL_FEATURE_SELECTION_CODER=True -KG_HYPOTHESIS_GEN=True \ No newline at end of file +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=False \ No newline at end of file diff --git a/scripts/exp/ablation/env/max.env b/scripts/exp/ablation/env/max.env index 40c67404..7dd10e10 100644 --- a/scripts/exp/ablation/env/max.env +++ b/scripts/exp/ablation/env/max.env @@ -1,5 +1,3 @@ KG_IF_USING_VECTOR_RAG=True KG_IF_USING_GRAPH_RAG=True -KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True -KG_MODEL_FEATURE_SELECTION_CODER=True -KG_HYPOTHESIS_GEN=True \ No newline at end of file +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True \ No newline at end of file diff --git a/scripts/exp/ablation/env/mini-case.env b/scripts/exp/ablation/env/mini-case.env index 87b0e716..d2ca7c4f 100644 --- a/scripts/exp/ablation/env/mini-case.env +++ b/scripts/exp/ablation/env/mini-case.env @@ -1,5 +1,3 @@ KG_IF_USING_VECTOR_RAG=True KG_IF_USING_GRAPH_RAG=False KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True -KG_MODEL_FEATURE_SELECTION_CODER=True -KG_HYPOTHESIS_GEN=False \ No newline at end of file diff --git a/scripts/exp/ablation/env/pro.env b/scripts/exp/ablation/env/pro.env index 2ba8cc0f..d2ca7c4f 100644 --- a/scripts/exp/ablation/env/pro.env +++ b/scripts/exp/ablation/env/pro.env @@ -1,5 +1,3 @@ KG_IF_USING_VECTOR_RAG=True KG_IF_USING_GRAPH_RAG=False KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True -KG_MODEL_FEATURE_SELECTION_CODER=True -KG_HYPOTHESIS_GEN=True \ No newline at end of file diff --git a/scripts/exp/tools/README.md b/scripts/exp/tools/README.md index 659fbf8b..bb0a8fd8 100644 --- a/scripts/exp/tools/README.md +++ b/scripts/exp/tools/README.md @@ -1,3 +1,138 @@ -The tools in the directory contains following generalfeatures -- collecting envs and run each -- collect results and generate summary +Tools Directory + +This directory provides scripts to run experiments with different environment configurations, collect results, and demonstrate usage through an example script. + +Directory Structure + +scripts/exp/tools/ +├── run_envs.sh # Script for running experiments +├── collect.py # Results collection and summary +├── test_system.sh # Example usage script +├── README.md # This documentation + +Tools Overview + + 1. run_envs.sh: Executes experiments with different environment configurations in parallel. + 2. collect.py: Collects and summarizes experiment results into a single file. + 3. test_system.sh: Demonstrates how to use the above tools together for experiment execution and result collection. + +Getting Started + +Prerequisites + + 1. Ensure the scripts have execution permissions: + +chmod +x scripts/exp/tools/run_envs.sh +chmod +x scripts/exp/tools/test_system.sh + + + 2. Install required dependencies for Python scripts: + +pip install -r requirements.txt + + + 3. Place your .env files in the desired directory for environment configurations. + +Usage + +1. Running Experiments with Different Environments + +The run_envs.sh script allows running a command with multiple environment configurations in parallel. + +Command Syntax + +./run_envs.sh -d -j -- + +Example Usage + +Basic example: + +./run_envs.sh -d env_files -j 1 -- echo "Hello" + +Practical example: + +dotenv run -- ./run_envs.sh -d /h/home/v-xisenwang/RD-Agent/scripts/exp/ablation/env -j 1 -- python /home/v-xisenwang/RD-Agent/rdagent/app/kaggle/loop.py + +Explanation: + • -d: Specifies the directory containing .env files. + • -j: Number of parallel processes to run (e.g., 1 for sequential execution). + • --: Separates script options from the command to execute. + • : The command to execute with the environment variables loaded. + +2. Collecting Results + +The collect.py script processes logs and generates a summary JSON file. + +Command Syntax + +python collect.py --log_path --output_name + +Example Usage + +Collect results from logs: + +python collect.py --log_path logs --output_name summary.json + +Explanation: + • --log_path: Required. Specifies the directory containing experiment logs. + • --output_name: Optional. The name of the output summary file (default: summary.json). + +3. Example Workflow + +Use the test_system.sh script to demonstrate a complete workflow. + +Steps: + + 1. Ensure the scripts are executable: + +chmod +x scripts/exp/tools/run_envs.sh +chmod +x scripts/exp/tools/test_system.sh + + + 2. Run the test system: + +./scripts/exp/tools/test_system.sh + + + +This will: + 1. Load environment configurations from .env files. + 2. Execute experiments using the configurations. + 3. Collect and summarize results. + +Troubleshooting + +Permission Denied + +If you encounter a PermissionError when running scripts: + 1. Ensure the script has execution permissions: + +chmod +x ./scripts/exp/tools/run_envs.sh +chmod +x ./scripts/exp/tools/test_system.sh + + + 2. Verify file ownership: + +ls -l ./scripts/exp/tools/ + + + 3. Change ownership or update permissions if necessary: + +sudo chown $USER:$USER ./scripts/exp/tools/* +chmod 755 ./scripts/exp/tools/* + +Capitalized Environment Variables + +To standardize variable names, .env files should use uppercase variable names with underscores (e.g., MY_VARIABLE). For example: + +IF_USING_VECTOR_RAG=true +IF_USING_GRAPH_RAG=false +MODEL_FEATURE_SELECTION_CODER=advanced + +Notes + + • Use the -j parameter to scale parallel processes as needed. + • Ensure .env files are correctly formatted to avoid errors. + • Customize test_system.sh as per your project’s requirements. + +For further assistance, refer to the comments within the scripts or reach out to the development team. \ No newline at end of file From 68ca63abc9c011de78433a81c6f1bd687a49fac1 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 25 Nov 2024 12:16:19 +0000 Subject: [PATCH 16/21] Updated README.md --- scripts/exp/tools/README.md | 132 +++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 61 deletions(-) diff --git a/scripts/exp/tools/README.md b/scripts/exp/tools/README.md index bb0a8fd8..cc21ab6c 100644 --- a/scripts/exp/tools/README.md +++ b/scripts/exp/tools/README.md @@ -1,138 +1,148 @@ -Tools Directory +### Tools Directory This directory provides scripts to run experiments with different environment configurations, collect results, and demonstrate usage through an example script. -Directory Structure +### Directory Structure scripts/exp/tools/ ├── run_envs.sh # Script for running experiments ├── collect.py # Results collection and summary -├── test_system.sh # Example usage script +├── test_system.sh # Usage script for rdagent kaggle loop ├── README.md # This documentation Tools Overview 1. run_envs.sh: Executes experiments with different environment configurations in parallel. 2. collect.py: Collects and summarizes experiment results into a single file. - 3. test_system.sh: Demonstrates how to use the above tools together for experiment execution and result collection. + 3. test_system.sh: Demonstrates how to use the above tools together for experiment execution and result collection [for rdagent kaggle loop] -Getting Started +### Getting Started Prerequisites - 1. Ensure the scripts have execution permissions: +1. Ensure the scripts have execution permissions: +``` chmod +x scripts/exp/tools/run_envs.sh chmod +x scripts/exp/tools/test_system.sh +``` +2. Place your .env files in the desired directory for environment configurations. - 2. Install required dependencies for Python scripts: +### Usage -pip install -r requirements.txt - - - 3. Place your .env files in the desired directory for environment configurations. - -Usage - -1. Running Experiments with Different Environments +#### 1. Running Experiments with Different Environments The run_envs.sh script allows running a command with multiple environment configurations in parallel. -Command Syntax +**Command Syntax** +``` ./run_envs.sh -d -j -- +``` -Example Usage +**Example Usage** Basic example: +``` ./run_envs.sh -d env_files -j 1 -- echo "Hello" +``` Practical example: -dotenv run -- ./run_envs.sh -d /h/home/v-xisenwang/RD-Agent/scripts/exp/ablation/env -j 1 -- python /home/v-xisenwang/RD-Agent/rdagent/app/kaggle/loop.py +``` +dotenv run -- ./run_envs.sh -d RD-Agent/scripts/exp/ablation/env -j 1 -- python RD-Agent/rdagent/app/kaggle/loop.py +``` + +**Explanation:** + +| Option | Description | +| --- | --- | +| `-d` | Specifies the directory containing .env files. | +| `-j` | Number of parallel processes to run (e.g., 1 for sequential execution). | +| `--` | Separates script options from the command to execute. | +| `` | The command to execute with the environment variables loaded. | -Explanation: - • -d: Specifies the directory containing .env files. - • -j: Number of parallel processes to run (e.g., 1 for sequential execution). - • --: Separates script options from the command to execute. - • : The command to execute with the environment variables loaded. -2. Collecting Results +####2.Collecting Results The collect.py script processes logs and generates a summary JSON file. -Command Syntax +**Command Syntax** +``` python collect.py --log_path --output_name +``` -Example Usage +**Example Usage** Collect results from logs: +``` python collect.py --log_path logs --output_name summary.json +``` +**Explanation:** -Explanation: - • --log_path: Required. Specifies the directory containing experiment logs. - • --output_name: Optional. The name of the output summary file (default: summary.json). +| Option | Description | +| --- | --- | +| `--log_path` | Required. Specifies the directory containing experiment logs. | +| `--output_name` | Optional. The name of the output summary file (default: summary.json). | -3. Example Workflow +#### 3. Example Workflow [for rdagent kaggle loop] Use the test_system.sh script to demonstrate a complete workflow. -Steps: +**Steps:** - 1. Ensure the scripts are executable: +1. Ensure the scripts are executable: +``` chmod +x scripts/exp/tools/run_envs.sh chmod +x scripts/exp/tools/test_system.sh +``` +2. Run the test system: - 2. Run the test system: - +``` ./scripts/exp/tools/test_system.sh - - +``` This will: 1. Load environment configurations from .env files. 2. Execute experiments using the configurations. - 3. Collect and summarize results. - -Troubleshooting - -Permission Denied - -If you encounter a PermissionError when running scripts: - 1. Ensure the script has execution permissions: - -chmod +x ./scripts/exp/tools/run_envs.sh -chmod +x ./scripts/exp/tools/test_system.sh +3. Find your logs in the logs directory. - 2. Verify file ownership: +4. Use the collect.py script to summarize results: -ls -l ./scripts/exp/tools/ +``` +python collect.py --log_path logs --output_name summary.json +``` +### Troubleshooting - 3. Change ownership or update permissions if necessary: +#### Permission Denied -sudo chown $USER:$USER ./scripts/exp/tools/* -chmod 755 ./scripts/exp/tools/* +If you encounter a PermissionError when running scripts: -Capitalized Environment Variables +1.Ensure the script has execution permissions: -To standardize variable names, .env files should use uppercase variable names with underscores (e.g., MY_VARIABLE). For example: +``` +chmod +x ./scripts/exp/tools/run_envs.sh +chmod +x ./scripts/exp/tools/test_system.sh +``` -IF_USING_VECTOR_RAG=true -IF_USING_GRAPH_RAG=false -MODEL_FEATURE_SELECTION_CODER=advanced +2.Verify file ownership: -Notes +``` +ls -l ./scripts/exp/tools/ +``` - • Use the -j parameter to scale parallel processes as needed. - • Ensure .env files are correctly formatted to avoid errors. - • Customize test_system.sh as per your project’s requirements. +### Notes +* Scale parallel processes as needed using the -j parameter. +* Avoid errors by ensuring .env files are correctly formatted. +* Modify test_system.sh to meet your project's specific needs. +* Add other metrics interested in collect.py to summarize automatically. For further assistance, refer to the comments within the scripts or reach out to the development team. \ No newline at end of file From 8b18fade6c6f246a750998579a607fd1c8b1c123 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 25 Nov 2024 12:21:38 +0000 Subject: [PATCH 17/21] reverting gitignore back --- .gitignore | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 7ec4ee96..400cf7d8 100644 --- a/.gitignore +++ b/.gitignore @@ -151,7 +151,7 @@ reports/ # git_ignore_folder git_ignore_folder/ -# cache +#cache *cache*/ *cache.json @@ -169,11 +169,4 @@ mlruns/ # shell script *.out - -# Logs -*.log -logs/ -log/ - -# Ignore results directory -RD-Agent/rdagent/scenarios/kaggle/automated_evaluation/results/ +*.sh From 2395dc5835cce33698096ecf605ad70ff20d9f5a Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Tue, 3 Dec 2024 03:05:00 +0000 Subject: [PATCH 18/21] Updates --- scripts/exp/ablation/README.md | 36 +++++++++++++++++++++++--- scripts/exp/ablation/env/max.env | 6 +++-- scripts/exp/ablation/env/mini-case.env | 1 + scripts/exp/tools/README.md | 2 +- scripts/exp/tools/test_system.sh | 6 ++--- 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/scripts/exp/ablation/README.md b/scripts/exp/ablation/README.md index 72b8c512..a5bec66b 100644 --- a/scripts/exp/ablation/README.md +++ b/scripts/exp/ablation/README.md @@ -1,9 +1,37 @@ # Introduction -| name | .env | desc | -| -- | -- | -- | -| full | full.env | enable all features | -| minicase | minicase.env | enable minicase | +This document outlines the environment configurations for the ablation studies. Each environment file corresponds to a specific experimental case, with some cases currently unavailable for implementation. + +| Name | .env | Description | Available? | +|-----------|--------------|-------------------------------------------|------------| +| max | max.env | Enables all features | No | +| minicase | minicase.env | Enables minicase and DS-Agent | Yes | +| pro | pro.env | Standard case with vector RAG | Yes | + +## Notes + +- Each `.env` file represents a distinct case for experimentation. Future implementations will include the unavailable cases. +- There is potential for integrating `CHAT_MODEL` in the future to facilitate comparisons between different models in experiments. + +## Common Environment Variables + +| Variable Name | Description | +|-----------------------------------|-----------------------------------------------------------------------------| +| `MINICASE` | Set to `True` to enable the previous implementation of DS-Agent. | +| `IF_USING_MLE_DATA` | Set to `True` to use MLE benchmark data; requires `KG_LOCAL_DATA_PATH=/data/userdata/share/mle_kaggle`. | +| `KG_IF_USING_VECTOR_RAG` | Set to `True` to enable vector RAG. | +| `KG_IF_USING_GRAPH_RAG` | Set to `False` to disable graph RAG. | +| `KG_IF_ACTION_CHOOSING_BASED_ON_UCB` | Set to `True` to enable action selection based on UCB. | + +## Future Work + +- Implement additional environment configurations as needed. +- Explore the integration of different models for comparative analysis in ablation studies. + + + + + diff --git a/scripts/exp/ablation/env/max.env b/scripts/exp/ablation/env/max.env index 7dd10e10..af3b5bd8 100644 --- a/scripts/exp/ablation/env/max.env +++ b/scripts/exp/ablation/env/max.env @@ -1,3 +1,5 @@ -KG_IF_USING_VECTOR_RAG=True +KG_IF_USING_VECTOR_RAG=False KG_IF_USING_GRAPH_RAG=True -KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True \ No newline at end of file +KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True +#KG_KNOWLEDGE_BASE_PATH= TODO: Specify Your Knowledge Base Path +CHAT_MODEL \ No newline at end of file diff --git a/scripts/exp/ablation/env/mini-case.env b/scripts/exp/ablation/env/mini-case.env index d2ca7c4f..dee1f6e3 100644 --- a/scripts/exp/ablation/env/mini-case.env +++ b/scripts/exp/ablation/env/mini-case.env @@ -1,3 +1,4 @@ KG_IF_USING_VECTOR_RAG=True KG_IF_USING_GRAPH_RAG=False KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True + diff --git a/scripts/exp/tools/README.md b/scripts/exp/tools/README.md index cc21ab6c..d1060a7f 100644 --- a/scripts/exp/tools/README.md +++ b/scripts/exp/tools/README.md @@ -49,7 +49,7 @@ Basic example: ./run_envs.sh -d env_files -j 1 -- echo "Hello" ``` -Practical example: +Practical example (running the kaggle loop file): ``` dotenv run -- ./run_envs.sh -d RD-Agent/scripts/exp/ablation/env -j 1 -- python RD-Agent/rdagent/app/kaggle/loop.py diff --git a/scripts/exp/tools/test_system.sh b/scripts/exp/tools/test_system.sh index 4b417cd5..465ac07b 100755 --- a/scripts/exp/tools/test_system.sh +++ b/scripts/exp/tools/test_system.sh @@ -5,9 +5,9 @@ TEST_DIR="test_run" mkdir -p "$TEST_DIR/results" mkdir -p "$TEST_DIR/logs" -# Define paths -ENV_DIR="/home/v-xisenwang/RD-Agent/scripts/exp/ablation/env" -PYTHON_SCRIPT="/home/v-xisenwang/RD-Agent/rdagent/app/kaggle/loop.py" +# Define relative paths inside the folder RDAgent +ENV_DIR="scripts/exp/ablation/env" # The folder of environments to apply +PYTHON_SCRIPT="rdagent/app/kaggle/loop.py" # The main file for running # Run the experiment echo "Running experiments..." From b7cc98eb1f6d6a2a301631a501050e23464b2c51 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Tue, 3 Dec 2024 03:12:26 +0000 Subject: [PATCH 19/21] README update --- scripts/exp/tools/README.md | 157 +++++++++++++++--------------------- 1 file changed, 67 insertions(+), 90 deletions(-) diff --git a/scripts/exp/tools/README.md b/scripts/exp/tools/README.md index d1060a7f..9dc4d4f6 100644 --- a/scripts/exp/tools/README.md +++ b/scripts/exp/tools/README.md @@ -1,148 +1,125 @@ -### Tools Directory +# Tools Directory This directory provides scripts to run experiments with different environment configurations, collect results, and demonstrate usage through an example script. -### Directory Structure +## Directory Structure +``` scripts/exp/tools/ ├── run_envs.sh # Script for running experiments ├── collect.py # Results collection and summary ├── test_system.sh # Usage script for rdagent kaggle loop -├── README.md # This documentation - -Tools Overview - - 1. run_envs.sh: Executes experiments with different environment configurations in parallel. - 2. collect.py: Collects and summarizes experiment results into a single file. - 3. test_system.sh: Demonstrates how to use the above tools together for experiment execution and result collection [for rdagent kaggle loop] +└── README.md # This documentation +``` -### Getting Started +## Tools Overview -Prerequisites +1. **run_envs.sh**: Executes experiments with different environment configurations in parallel. +2. **collect.py**: Collects and summarizes experiment results into a single file. +3. **test_system.sh**: Demonstrates how to use the above tools together for experiment execution and result collection (for rdagent kaggle loop). -1. Ensure the scripts have execution permissions: +## Getting Started -``` -chmod +x scripts/exp/tools/run_envs.sh -chmod +x scripts/exp/tools/test_system.sh -``` +### Prerequisites -2. Place your .env files in the desired directory for environment configurations. +Place your `.env` files in the desired directory for environment configurations. -### Usage +## Usage -#### 1. Running Experiments with Different Environments +### 1. Running Experiments with Different Environments -The run_envs.sh script allows running a command with multiple environment configurations in parallel. +The `run_envs.sh` script allows running a command with multiple environment configurations in parallel. -**Command Syntax** +**Command Syntax:** -``` +```bash ./run_envs.sh -d -j -- ``` -**Example Usage** +**Example Usage:** -Basic example: +- Basic example: -``` -./run_envs.sh -d env_files -j 1 -- echo "Hello" -``` + ```bash + ./run_envs.sh -d env_files -j 1 -- echo "Hello" + ``` -Practical example (running the kaggle loop file): +- Practical example (running the kaggle loop file): -``` -dotenv run -- ./run_envs.sh -d RD-Agent/scripts/exp/ablation/env -j 1 -- python RD-Agent/rdagent/app/kaggle/loop.py -``` + ```bash + dotenv run -- ./run_envs.sh -d RD-Agent/scripts/exp/ablation/env -j 1 -- python RD-Agent/rdagent/app/kaggle/loop.py + ``` **Explanation:** -| Option | Description | -| --- | --- | -| `-d` | Specifies the directory containing .env files. | -| `-j` | Number of parallel processes to run (e.g., 1 for sequential execution). | -| `--` | Separates script options from the command to execute. | -| `` | The command to execute with the environment variables loaded. | - +| Option | Description | +|-------------|--------------------------------------------------------------| +| `-d` | Specifies the directory containing `.env` files. | +| `-j` | Number of parallel processes to run (e.g., 1 for sequential execution). | +| `--` | Separates script options from the command to execute. | +| ``| The command to execute with the environment variables loaded.| -####2.Collecting Results +### 2. Collecting Results -The collect.py script processes logs and generates a summary JSON file. +The `collect.py` script processes logs and generates a summary JSON file. -**Command Syntax** +**Command Syntax:** -``` +```bash python collect.py --log_path --output_name ``` -**Example Usage** +**Example Usage:** Collect results from logs: -``` +```bash python collect.py --log_path logs --output_name summary.json ``` + **Explanation:** -| Option | Description | -| --- | --- | -| `--log_path` | Required. Specifies the directory containing experiment logs. | -| `--output_name` | Optional. The name of the output summary file (default: summary.json). | +| Option | Description | +|-----------------|--------------------------------------------------------------| +| `--log_path` | Required. Specifies the directory containing experiment logs.| +| `--output_name`| Optional. The name of the output summary file (default: summary.json). | -#### 3. Example Workflow [for rdagent kaggle loop] +### 3. Example Workflow (for rdagent kaggle loop) -Use the test_system.sh script to demonstrate a complete workflow. +Use the `test_system.sh` script to demonstrate a complete workflow. **Steps:** -1. Ensure the scripts are executable: +1. Run the test system: -``` -chmod +x scripts/exp/tools/run_envs.sh -chmod +x scripts/exp/tools/test_system.sh -``` - -2. Run the test system: - -``` -./scripts/exp/tools/test_system.sh -``` + ```bash + ./scripts/exp/tools/test_system.sh + ``` -This will: - 1. Load environment configurations from .env files. - 2. Execute experiments using the configurations. + This will: + 1. Load environment configurations from `.env` files. + 2. Execute experiments using the configurations. -3. Find your logs in the logs directory. +2. Find your logs in the `logs` directory. -4. Use the collect.py script to summarize results: +3. Use the `collect.py` script to summarize results: -``` -python collect.py --log_path logs --output_name summary.json -``` + ```bash + python collect.py --log_path logs --output_name summary.json + ``` -### Troubleshooting +## Create Your Own Workflow -#### Permission Denied +- Create the ablation environments under a specified folder. +- Revise the `test_system.sh` template to adjust the path and relevant commands for execution. +- Run `test_system.sh` to execute the environments through different configurations. +- Keep track of your log path and use `collect.py` to collect the results at scale. -If you encounter a PermissionError when running scripts: - -1.Ensure the script has execution permissions: - -``` -chmod +x ./scripts/exp/tools/run_envs.sh -chmod +x ./scripts/exp/tools/test_system.sh -``` - -2.Verify file ownership: - -``` -ls -l ./scripts/exp/tools/ -``` +## Notes -### Notes -* Scale parallel processes as needed using the -j parameter. -* Avoid errors by ensuring .env files are correctly formatted. -* Modify test_system.sh to meet your project's specific needs. -* Add other metrics interested in collect.py to summarize automatically. +- Scale parallel processes as needed using the `-j` parameter. +- Avoid errors by ensuring `.env` files are correctly formatted. +- Modify `test_system.sh` to meet your project's specific needs. +- Add other metrics of interest in `collect.py` to summarize automatically. For further assistance, refer to the comments within the scripts or reach out to the development team. \ No newline at end of file From 0b5a09d1eec7ebf462d779e6c3f7188a703d8d9b Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Tue, 3 Dec 2024 03:15:30 +0000 Subject: [PATCH 20/21] Updates on env README --- scripts/exp/ablation/README.md | 5 +++-- scripts/exp/ablation/env/max.env | 1 - scripts/exp/ablation/env/mini-case.env | 1 + scripts/exp/ablation/env/pro.env | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/exp/ablation/README.md b/scripts/exp/ablation/README.md index a5bec66b..ac20a8cb 100644 --- a/scripts/exp/ablation/README.md +++ b/scripts/exp/ablation/README.md @@ -4,9 +4,10 @@ This document outlines the environment configurations for the ablation studies. | Name | .env | Description | Available? | |-----------|--------------|-------------------------------------------|------------| -| max | max.env | Enables all features | No | -| minicase | minicase.env | Enables minicase and DS-Agent | Yes | +| basic | basic.env | Standard case of RDAgent | Yes | +| minicase | minicase.env | Enables minicase and DS-Agent | Yes | | pro | pro.env | Standard case with vector RAG | Yes | +| max | max.env | Enables all features | No | ## Notes diff --git a/scripts/exp/ablation/env/max.env b/scripts/exp/ablation/env/max.env index af3b5bd8..575b5aa3 100644 --- a/scripts/exp/ablation/env/max.env +++ b/scripts/exp/ablation/env/max.env @@ -2,4 +2,3 @@ KG_IF_USING_VECTOR_RAG=False KG_IF_USING_GRAPH_RAG=True KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True #KG_KNOWLEDGE_BASE_PATH= TODO: Specify Your Knowledge Base Path -CHAT_MODEL \ No newline at end of file diff --git a/scripts/exp/ablation/env/mini-case.env b/scripts/exp/ablation/env/mini-case.env index dee1f6e3..91998774 100644 --- a/scripts/exp/ablation/env/mini-case.env +++ b/scripts/exp/ablation/env/mini-case.env @@ -1,4 +1,5 @@ KG_IF_USING_VECTOR_RAG=True KG_IF_USING_GRAPH_RAG=False KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True +# MIGHT BE LEGACY diff --git a/scripts/exp/ablation/env/pro.env b/scripts/exp/ablation/env/pro.env index d2ca7c4f..67635788 100644 --- a/scripts/exp/ablation/env/pro.env +++ b/scripts/exp/ablation/env/pro.env @@ -1,3 +1,4 @@ KG_IF_USING_VECTOR_RAG=True KG_IF_USING_GRAPH_RAG=False KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True +# MIGHT BE LEGACY From 24cd0c24e12627cad22f149e4688d8e885ab186c Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Tue, 3 Dec 2024 04:15:13 +0000 Subject: [PATCH 21/21] Updating collect.py --- scripts/exp/tools/collect.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/exp/tools/collect.py b/scripts/exp/tools/collect.py index 8ab2d7a2..001d0365 100644 --- a/scripts/exp/tools/collect.py +++ b/scripts/exp/tools/collect.py @@ -7,6 +7,7 @@ from rdagent.scenarios.kaggle.kaggle_crawler import ( leaderboard_scores, ) +import pandas as pd def collect_results(log_path) -> list[dict]: summary = [] @@ -39,24 +40,32 @@ def generate_summary(results, output_path): } for result in results: # Update best result - # If the evaluation metric is higher, it is better if result["evaluation_metric_direction"]: if (result["score"] is not None and (summary["best_result"]["score"] is None or - result["score"] > summary["best_result"]["score"])): + (result["score"].iloc[0] > summary["best_result"]["score"]))): summary["best_result"].update({ - "score": result["score"], + "score": result["score"].iloc[0] if isinstance(result["score"], pd.Series) else result["score"], "competition_name": result["competition_name"] }) else: if (result["score"] is not None and (summary["best_result"]["score"] is None or - result["score"] < summary["best_result"]["score"])): + (result["score"].iloc[0] < summary["best_result"]["score"]))): summary["best_result"].update({ - "score": result["score"], + "score": result["score"].iloc[0] if isinstance(result["score"], pd.Series) else result["score"], "competition_name": result["competition_name"] }) + # Convert Series to scalar or list if necessary + for key, value in summary.items(): + if isinstance(value, pd.Series): + summary[key] = value.tolist() # Convert Series to list + elif isinstance(value, dict): + for sub_key, sub_value in value.items(): + if isinstance(sub_value, pd.Series): + value[sub_key] = sub_value.tolist() # Convert Series to list + with open(output_path, "w") as f: json.dump(summary, f, indent=4)