From 89d104d8fe02af641d0f00d3783df8f9c6e215d2 Mon Sep 17 00:00:00 2001 From: Gerald Woo Date: Tue, 11 Jun 2024 11:18:42 +0800 Subject: [PATCH 1/4] Create lint.yml --- .github/workflows/lint.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..5ed748e --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,21 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Lint + +on: [push, pull_request] + +jobs: + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable + with: + use_pyproject: true + jupyter: true + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: isort/isort-action@v1 From adf72061666813456200f3bf083c8389eaca4bfe Mon Sep 17 00:00:00 2001 From: Gerald Woo Date: Tue, 11 Jun 2024 13:41:45 +0800 Subject: [PATCH 2/4] Add GitHub Actions (#70) * Update lint.yml * Add test workflow --- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5ed748e..10a8040 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - uses: psf/black@stable with: - use_pyproject: true + options: "--check --verbose --line-length 88" jupyter: true isort: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a88a881 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: Test + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e '.[dev,notebook]' + - name: Test with pytest + run: | + pytest test From 70ed963781c0c94a56b0066e483a5ee4b931ea8f Mon Sep 17 00:00:00 2001 From: Juncheng Liu Date: Fri, 14 Jun 2024 16:39:13 +0800 Subject: [PATCH 3/4] Evaluate chronos on monash Scripts and code to evaluate chronos on monash. --- project/benchmarks/README.md | 14 ++ .../chronos_scripts/monash_chronos_base.sh | 6 + .../chronos_scripts/monash_chronos_mini.sh | 6 + .../chronos_scripts/monash_chronos_small.sh | 6 + .../chronos_scripts/monash_chronos_tiny.sh | 6 + project/benchmarks/run_chronos.py | 125 ++++++++++++++++++ 6 files changed, 163 insertions(+) create mode 100644 project/benchmarks/README.md create mode 100644 project/benchmarks/chronos_scripts/monash_chronos_base.sh create mode 100644 project/benchmarks/chronos_scripts/monash_chronos_mini.sh create mode 100644 project/benchmarks/chronos_scripts/monash_chronos_small.sh create mode 100644 project/benchmarks/chronos_scripts/monash_chronos_tiny.sh create mode 100644 project/benchmarks/run_chronos.py diff --git a/project/benchmarks/README.md b/project/benchmarks/README.md new file mode 100644 index 0000000..36d3276 --- /dev/null +++ b/project/benchmarks/README.md @@ -0,0 +1,14 @@ +# Benchmark +This directory contains the code and scripts for benchmarking. + + +## Chronos +`run_chronos.py` is the code to run Chronos on a given dataset. + +`chronos_scripts` contains the scripts to run Chronos on different datasets. + +Example: +``` +sh chronos_scripts/monash_chronos_base.sh +``` + diff --git a/project/benchmarks/chronos_scripts/monash_chronos_base.sh b/project/benchmarks/chronos_scripts/monash_chronos_base.sh new file mode 100644 index 0000000..0d7a527 --- /dev/null +++ b/project/benchmarks/chronos_scripts/monash_chronos_base.sh @@ -0,0 +1,6 @@ +model_size=base +model_path=amazon/chronos-t5-${model_size} +for ds in us_births saugeenday sunspot_with_missing temperature_rain_with_missing covid_deaths hospital rideshare_with_missing traffic_weekly traffic_hourly fred_md car_parts_with_missing electricity_weekly electricity_hourly solar_weekly solar_10_minutes nn5_weekly nn5_daily_with_missing weather kdd_cup_2018_with_missing vehicle_trips_with_missing pedestrian_counts bitcoin_with_missing dominick australian_electricity_demand cif_2016_12 cif_2016_6 tourism_monthly tourism_quarterly m4_hourly m4_daily m4_weekly m4_monthly monash_m3_other monash_m3_monthly m1_monthly m1_yearly monash_m3_yearly m4_yearly tourism_yearly m1_quarterly monash_m3_quarterly m4_quarterly kaggle_web_traffic_weekly kaggle_web_traffic_with_missing bitcoin +do + python run_chronos.py --model_path=${model_path} --dataset=${ds} --run_name=chronos-${model_size} +done \ No newline at end of file diff --git a/project/benchmarks/chronos_scripts/monash_chronos_mini.sh b/project/benchmarks/chronos_scripts/monash_chronos_mini.sh new file mode 100644 index 0000000..bc40089 --- /dev/null +++ b/project/benchmarks/chronos_scripts/monash_chronos_mini.sh @@ -0,0 +1,6 @@ +model_size=mini +model_path=amazon/chronos-t5-${model_size} +for ds in us_births saugeenday sunspot_with_missing temperature_rain_with_missing covid_deaths hospital rideshare_with_missing traffic_weekly traffic_hourly fred_md car_parts_with_missing electricity_weekly electricity_hourly solar_weekly solar_10_minutes nn5_weekly nn5_daily_with_missing weather kdd_cup_2018_with_missing vehicle_trips_with_missing pedestrian_counts bitcoin_with_missing dominick australian_electricity_demand cif_2016_12 cif_2016_6 tourism_monthly tourism_quarterly m4_hourly m4_daily m4_weekly m4_monthly monash_m3_other monash_m3_monthly m1_monthly m1_yearly monash_m3_yearly m4_yearly tourism_yearly m1_quarterly monash_m3_quarterly m4_quarterly kaggle_web_traffic_weekly kaggle_web_traffic_with_missing bitcoin +do + python run_chronos.py --model_path=${model_path} --dataset=${ds} --run_name=chronos-${model_size} +done \ No newline at end of file diff --git a/project/benchmarks/chronos_scripts/monash_chronos_small.sh b/project/benchmarks/chronos_scripts/monash_chronos_small.sh new file mode 100644 index 0000000..bd76a52 --- /dev/null +++ b/project/benchmarks/chronos_scripts/monash_chronos_small.sh @@ -0,0 +1,6 @@ +model_size=small +model_path=amazon/chronos-t5-${model_size} +for ds in us_births saugeenday sunspot_with_missing temperature_rain_with_missing covid_deaths hospital rideshare_with_missing traffic_weekly traffic_hourly fred_md car_parts_with_missing electricity_weekly electricity_hourly solar_weekly solar_10_minutes nn5_weekly nn5_daily_with_missing weather kdd_cup_2018_with_missing vehicle_trips_with_missing pedestrian_counts bitcoin_with_missing dominick australian_electricity_demand cif_2016_12 cif_2016_6 tourism_monthly tourism_quarterly m4_hourly m4_daily m4_weekly m4_monthly monash_m3_other monash_m3_monthly m1_monthly m1_yearly monash_m3_yearly m4_yearly tourism_yearly m1_quarterly monash_m3_quarterly m4_quarterly kaggle_web_traffic_weekly kaggle_web_traffic_with_missing bitcoin +do + python run_chronos.py --model_path=${model_path} --dataset=${ds} --run_name=chronos-${model_size} +done \ No newline at end of file diff --git a/project/benchmarks/chronos_scripts/monash_chronos_tiny.sh b/project/benchmarks/chronos_scripts/monash_chronos_tiny.sh new file mode 100644 index 0000000..31935e6 --- /dev/null +++ b/project/benchmarks/chronos_scripts/monash_chronos_tiny.sh @@ -0,0 +1,6 @@ +model_size=tiny +model_path=amazon/chronos-t5-${model_size} +for ds in us_births saugeenday sunspot_with_missing temperature_rain_with_missing covid_deaths hospital rideshare_with_missing traffic_weekly traffic_hourly fred_md car_parts_with_missing electricity_weekly electricity_hourly solar_weekly solar_10_minutes nn5_weekly nn5_daily_with_missing weather kdd_cup_2018_with_missing vehicle_trips_with_missing pedestrian_counts bitcoin_with_missing dominick australian_electricity_demand cif_2016_12 cif_2016_6 tourism_monthly tourism_quarterly m4_hourly m4_daily m4_weekly m4_monthly monash_m3_other monash_m3_monthly m1_monthly m1_yearly monash_m3_yearly m4_yearly tourism_yearly m1_quarterly monash_m3_quarterly m4_quarterly kaggle_web_traffic_weekly kaggle_web_traffic_with_missing bitcoin +do + python run_chronos.py --model_path=${model_path} --dataset=${ds} --run_name=chronos-${model_size} +done \ No newline at end of file diff --git a/project/benchmarks/run_chronos.py b/project/benchmarks/run_chronos.py new file mode 100644 index 0000000..42870b2 --- /dev/null +++ b/project/benchmarks/run_chronos.py @@ -0,0 +1,125 @@ +import argparse +import os + +import numpy as np +import torch +from chronos import ChronosPipeline +from gluonts.dataset.repository import get_dataset +from gluonts.dataset.split import split +from gluonts.ev.metrics import ( + MAE, + MAPE, + MASE, + MSE, + MSIS, + ND, + NRMSE, + RMSE, + SMAPE, + MeanWeightedSumQuantileLoss, +) +from gluonts.itertools import batcher + +# from gluonts.model.evaluation import evaluate_forecasts +from gluonts.model.forecast import SampleForecast +from tqdm.auto import tqdm + +from uni2ts.eval_util.data import get_gluonts_test_dataset +from uni2ts.eval_util.evaluation import evaluate_forecasts +from uni2ts.eval_util.metrics import MedianMSE + + +def evaluate(pipeline, dataset, save_path, num_samples=20, batch_size=512): + print("-" * 5, f"Evaluating {dataset}", "-" * 5) + test_data, metadata = get_gluonts_test_dataset(dataset) + prediction_length = metadata.prediction_length + + while True: + try: + # Generate forecast samples + forecast_samples = [] + for batch in tqdm(batcher(test_data.input, batch_size=batch_size)): + context = [torch.tensor(entry["target"]) for entry in batch] + forecast_samples.append( + pipeline.predict( + context, + prediction_length=prediction_length, + num_samples=num_samples, + limit_prediction_length=False, # We disable the limit on prediction length. + ).numpy() + ) + forecast_samples = np.concatenate(forecast_samples) + break + except torch.cuda.OutOfMemoryError: + print( + f"OutOfMemoryError at batch_size {batch_size}, reducing to {batch_size//2}" + ) + batch_size //= 2 + + # Convert forecast samples into gluonts SampleForecast objects + sample_forecasts = [] + for item, ts in zip(forecast_samples, test_data.input): + forecast_start_date = ts["start"] + len(ts["target"]) + sample_forecasts.append( + SampleForecast(samples=item, start_date=forecast_start_date) + ) + + # Evaluate + metrics_df = evaluate_forecasts( + sample_forecasts, + test_data=test_data, + metrics=[ + MSE(), + MAE(), + MAPE(), + SMAPE(), + MSIS(), + RMSE(), + NRMSE(), + ND(), + MASE(), + MedianMSE(), + MeanWeightedSumQuantileLoss(np.arange(0.1, 1.0, 0.1)), + ], + ) + metrics_df.index = [dataset] + print(metrics_df) + metrics_df.to_csv(save_path) + print(f"Results saved to {save_path}") + print("-" * 5, f"Evaluation of {dataset} complete", "-" * 5) + return metrics_df + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Load a model and dataset, then make predictions." + ) + parser.add_argument( + "--model_path", type=str, required=True, help="Path to load the model" + ) + parser.add_argument( + "--dataset", type=str, required=True, help="Name of the dataset to use" + ) + parser.add_argument( + "--save_dir", type=str, default="results", help="Directory to save the results" + ) + parser.add_argument( + "--num_samples", type=int, default=20, help="Number of samples to generate" + ) + parser.add_argument( + "--batch_size", type=int, default=512, help="Batch size for generating samples" + ) + parser.add_argument("--run_name", type=str, default="test", help="Name of the run") + + args = parser.parse_args() + # Load Chronos + pipeline = ChronosPipeline.from_pretrained( + # "amazon/chronos-t5-small", + args.model_path, + device_map="cuda:0", + torch_dtype=torch.bfloat16, + ) + output_dir = os.path.join(args.save_dir, args.run_name) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + evaluate(pipeline, args.dataset, os.path.join(output_dir, f"{args.dataset}.csv")) From 92ddd746d4274813249ab92deac094710b7b6a2f Mon Sep 17 00:00:00 2001 From: Juncheng Liu Date: Fri, 14 Jun 2024 16:38:32 +0800 Subject: [PATCH 4/4] Change to python3 for black pre-commit --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93d1c24..8f6f6ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,9 @@ repos: rev: 24.2.0 hooks: - id: black - language_version: python3.10 + language_version: python3 - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: - id: isort - name: isort (python) \ No newline at end of file + name: isort (python)