Parallel download of GOES images

dates_eurec4a.txt

@@ -0,0 +1,32 @@
+20200120
+20200121
+20200122
+20200123
+20200124
+20200125
+20200126
+20200127
+20200128
+20200129
+20200130
+20200131
+20200201
+20200202
+20200203
+20200204
+20200205
+20200206
+20200207
+20200208
+20200209
+20200210
+20200211
+20200212
+20200213
+20200214
+20200215
+20200216
+20200217
+20200218
+20200219
+20200220

scripts/goes_movies.py

@@ -0,0 +1,37 @@
+import imageio
+import glob
+from datetime import datetime, timedelta
+
+def video_from_snapshots(snapshot_folder, save_path, fps=8):
+    '''
+    Make a video from all images in a folder and save to path.
+    '''
+
+    print(f'Making video of snapshots in {snapshot_folder} ...')
+
+    writer = imageio.get_writer(save_path, fps=fps)
+
+    image_path = sorted(glob.glob(snapshot_folder+'goes16_ABI*'))
+
+    for im in image_path:
+        writer.append_data(imageio.imread(im))
+    writer.close()
+
+    print('video saved!')
+
+
+# Make list of dates for movies
+start_date = datetime(2020, 1, 20)
+end_date = datetime(2020, 2, 20)
+
+days_for_videos = [(start_date + timedelta(days=i)).strftime('%Y-%m-%d') for i in range((end_date - start_date).days + 1)]
+
+# Choose channel
+channel = 2
+
+# Make and save movies
+for day in days_for_videos:
+
+    video_from_snapshots(snapshot_folder=f"/scratch/m/m300931/data/GOES-16/snapshots/channel_{channel}/{day}/",
+                         save_path=f"/scratch/m/m300931/data/GOES-16/movies/channel_{channel}/goes16_ABI-L2-CMIPF_ch{channel}_{day}.mp4")
+

scripts/mapper_goes16.py

@@ -0,0 +1,67 @@
+import sys
+sys.path.append('..')
+
+from datetime import datetime
+import time
+import resource
+import os
+import argparse
+from itertools import chain
+
+from src.gogoesgone import processing as pr
+from src.gogoesgone import zarr_access as za
+
+# Satellite specifications
+channel = 13
+satellite = "goes16"
+product="ABI-L2-CMIPF"
+savepath = f"/scratch/m/m300931/mappers/GOES-16/channel_{channel}/"
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("date")
+    args = parser.parse_args()
+
+    print(args.date)
+
+    starttime = time.time()
+    print("Processing started ", datetime.now())
+
+    filename = savepath + f"{args.date}_{satellite}_{product}_{channel}.json"
+
+    if os.path.isfile(filename):
+        print("File already exists:", filename)
+        return
+
+    date_object = datetime.strptime(args.date, '%Y%m%d')
+    year = date_object.year
+    day_of_year = date_object.timetuple().tm_yday
+
+    print("Generating file list...")
+
+    if channel == 2:
+        # get only daylight hours
+        hour_range = [10,22] # in UTC time
+        flist = []
+        for hour in range(hour_range[0], hour_range[-1]):
+            gs = za.generate_globsearch_string(year, day_of_year, hour, channel, product, satellite)
+            flist.append(za.generate_url_list(gs))
+        flist = sorted(list(chain(*flist)))
+
+    elif channel == 13:
+        hour = None # None for all hours
+        gs = za.generate_globsearch_string(year, day_of_year, hour, channel, product, satellite)
+        flist = za.generate_url_list(gs)
+
+    if flist == None:
+        return
+
+    print(f"Mapping...")
+    za.get_mzz_from_references(flist, save=True, save_file=filename + ".partial")
+    os.rename(filename + ".partial", filename)
+    print(f'Mapping done for {args.date}!')
+    print("time:", round((time.time() - starttime)/60), 'min')
+    print("memory usage:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, "kB")
+
+if __name__ == "__main__":
+    main()

scripts/plot_goes16.py

@@ -0,0 +1,92 @@
+import cartopy.crs as ccrs
+import cartopy.feature as cfeature
+from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mticker
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+import numpy as np
+import xarray as xr
+import argparse
+import pandas as pd
+from datetime import datetime, date, timedelta
+import resource
+import os
+import time
+import yaml
+import eurec4a
+from collections import defaultdict
+from functools import reduce
+
+
+def main():
+
+    """
+    Plot satellite images, optionally with HALO trajectories. Using code from HALO-DROPS.
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("date")
+    args = parser.parse_args()
+
+    print(f"Plotting satellite images for {args.date}")
+
+    ################### USER specifies satellite and path parameters ###################
+
+    channel = 13
+    product = "ABI-L2-CMIPF"
+    satellite = "goes16"
+    extent = (-60.7,-54,11.3,15)
+
+    starttime = time.time()
+    print("Processing started ", datetime.now())
+
+    # Plotting parameters
+    if channel == 2:
+        cmin = 0
+        cmax = 0.2
+        use_cmap = "Greys_r"
+        save_path = "/work/mh0010/m300931/GOES-16/snapshots/channel_2/"
+
+    elif channel == 13:
+        cmin = 280
+        cmax = 297
+        use_cmap = "Greys"
+        save_path = "/work/mh0010/m300931/GOES-16/snapshots/channel_13/"
+
+    # Load subset data to plot
+    subset_filepath = f"/work/mh0010/m300931/GOES-16/subsets/channel_{channel}/CMI_subset_{args.date}_{satellite}_{product}_{channel}.nc"
+    subset = xr.open_dataset(subset_filepath)
+
+    ################### USER specifies satellite and path parameters ###################
+
+    for i, t in enumerate(subset.t):
+
+        print(t.values)
+
+        save_file = f"{satellite}_{product}_ch{channel}_{np.datetime_as_string(t.values, unit='s')}.png"
+
+        # Check if file already exists
+        if os.path.isfile(save_path+save_file):
+            print("File already exists:", save_file)
+            continue
+
+        fig = plt.figure(figsize=(8,5))
+        ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
+
+        # Plot satellite image
+        im_sat = subset.CMI.isel(t=i).plot(ax=ax,x='lon',y='lat',cmap=use_cmap,add_colorbar=False,vmin=cmin,vmax=cmax)
+        ax.set_extent(extent, crs=ccrs.PlateCarree())
+        ax.coastlines(resolution='10m', color='red', linewidth=1)
+
+        plt.title(t.values)
+        plt.show()
+
+        plt.savefig(save_path+save_file, dpi=300, bbox_inches="tight")
+        plt.close()
+
+    # Check time and memory urage
+    print("time:", (time.time() - starttime)/60, "min")
+    print("memory usage:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, "Kb")
+
+if __name__ == "__main__":
+    main()

scripts/run_mapper_goes16.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH --partition=compute
+#SBATCH --account=mh0010
+#SBATCH --job-name=goes_mapper
+#SBATCH --time=08:00:00
+#SBATCH --output=log/%x.%j.log
+#SBATCH --error=log/%x.%j.log
+#SBATCH --mail-user=nina.robbins@mpimet.mpg.de
+
+. ~/.bashrc
+mamba activate halodrops_env
+
+chmod +x mapper_goes16.py
+
+date_file="$1"
+
+# Get the number of CPU cores
+#NUM_CORES=$(nproc)
+#--cpus-per-task=16
+NUM_CORES=128
+
+# Use xargs to run the Python script in parallel for each argument from the input file
+cat "$date_file" | xargs -n 1 -P "$NUM_CORES" python -u mapper_goes16.py > log/mapper_goes16.txt
+
+mamba deactivate

scripts/run_plot_goes16.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --partition=compute
+#SBATCH --account=mh0010
+#SBATCH --job-name=goes_plot
+#SBATCH --nodes=1
+#SBATCH --time=08:00:00
+#SBATCH --mail-type=ALL
+#SBATCH --mail-user=nina.robbins@mpimet.mpg.de
+#SBATCH --output=log/%x.%j.log
+#SBATCH --error=log/%x.%j.log
+
+. ~/.bashrc
+mamba activate halodrops_env
+
+date_file="$1"
+
+# Get the number of CPU cores
+#NUM_CORES=$(nproc)
+NUM_CORES=16
+
+# Use xargs to run the Python script in parallel for each argument from the input file
+cat "$date_file" | xargs -n 1 -P "$NUM_CORES" python -u plot_goes16.py > log/plot_goes16.txt
+
+mamba deactivate

scripts/run_save_nc_goes16.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#SBATCH --partition=compute
+#SBATCH --account=mh0010
+#SBATCH --job-name=goes_data
+#SBATCH --time=05:00:00
+#SBATCH --output=log/%x.%j.log
+#SBATCH --error=log/%x.%j.log
+#SBATCH --mail-user=nina.robbins@mpimet.mpg.de
+
+. ~/.bashrc
+mamba activate halodrops_env
+
+date_file="$1"
+
+# Get the number of CPU cores
+#NUM_CORES=$(nproc)
+NUM_CORES=16
+
+# Use xargs to run the Python script in parallel for each argument from the input file
+cat "$date_file" | xargs -n 1 -P "$NUM_CORES" python -u save_nc_goes16.py > log/save_nc_goes16.txt
+
+mamba deactivate