copy_data.run

#!/bin/bash

# Cd into our working directory in case we're not into it already
cd "$(dirname "$0")";

echo "----------------------------------------------------------------------------------------------"
echo "icon-forecasts: Starting processing of icon-eu model data - `date`"
echo "----------------------------------------------------------------------------------------------"

# Folder to be used to download and process data
export MODEL_DATA_FOLDER="/home/ekman/ssd/guido/icon-eu/"
export HOME_FOLDER=$(pwd)
export N_CONCUR_PROCESSES=12 # not used anymore as it needs to be tuned per case
export NCFTP_BOOKMARK="mid"
DATA_DOWNLOAD=true
DATA_PLOTTING=true
DATA_UPLOAD=true

##### LOAD functions to download model data
. ./functions_download_dwd.sh
export SHELL=$(type -p bash)
# We need to open many files at the same time
ulimit -Sn 8192
# 
###########################################

# Retrieve run ##########################
latest_run=`python get_last_run.py`
if [ -f $MODEL_DATA_FOLDER/last_processed_run.txt ]; then
	latest_processed_run=`while read line; do echo $line; done < $MODEL_DATA_FOLDER/last_processed_run.txt`
	if [ $latest_run -gt $latest_processed_run ]; then
		echo "New run ${latest_run} found! Last processed run was ${latest_processed_run}."
	else
		echo "Run ${latest_processed_run} already processed, exiting"
		exit 0
	fi
fi

export year=${latest_run:0:4} 
export month=${latest_run:4:2} 
export day=${latest_run:6:2} 
export run=${latest_run:8:2}

# Skip runs that we don't want to process
# if [ $run -eq "09" ] || [ $run -eq "15" ] || [ $run -eq "21" ] || [ $run -eq "03" ]; then
# 	echo "Skipping run ${latest_run}"
# 	exit 0
# fi
#############################################


mkdir -p ${MODEL_DATA_FOLDER}it
mkdir -p ${MODEL_DATA_FOLDER}de
# Move to the data folder to do processing
cd ${MODEL_DATA_FOLDER} || { echo 'Cannot change to DATA folder' ; exit 1; }

# SECTION 1 - DATA DOWNLOAD ############################################################

if [ "$DATA_DOWNLOAD" = true ]; then
	echo "----------------------------------------------------------------------------------------------"
	echo "icon-forecasts: Starting downloading of data - `date`"
	echo "----------------------------------------------------------------------------------------------"
	# Remove older files
	rm ${MODEL_DATA_FOLDER}*.nc
	rm ${MODEL_DATA_FOLDER}*.grib2

	# Invariant
	download_invariant_icon_eu

	#2-D variables
	variables=("T_2M" "TD_2M" "U_10M" "V_10M" "PMSL" "CAPE_ML" "VMAX_10M" "TOT_PREC"\
				"CLCL" "CLCH" "CLCT" "SNOWLMT" "HZEROCL" "H_SNOW" "SNOW_GSP"\
				"SNOW_CON" "RAIN_GSP" "RAIN_CON" "TMAX_2M" "TMIN_2M" "WW" "SYNMSG_BT_CL_IR10.8")
	parallel -j 4 --delay 1 download_merge_2d_variable_icon_eu ::: "${variables[@]}"

	#3-D variables on pressure levels
	variables=("T" "FI" "RELHUM" "U" "V" "CLC")
	parallel -j 4 --delay 2 download_merge_3d_variable_icon_eu ::: "${variables[@]}"

	#soil levelsls
	download_merge_soil_variable_icon_eu W_SO
fi 

############################################################


# SECTION 2 - DATA PLOTTING ############################################################

if [ "$DATA_PLOTTING" = true ]; then
	echo "----------------------------------------------------------------------------------------------"
	echo "icon-forecasts: Starting plotting of data - `date`"
	echo "----------------------------------------------------------------------------------------------"
	python --version
	cp ${HOME_FOLDER}/plotting/*.py ${MODEL_DATA_FOLDER}

	export QT_QPA_PLATFORM=offscreen # Needed to avoid errors when using Python without display

	python plot_meteogram.py Hamburg Pisa Rome Milano Naples Palermo Toulouse Utrecht Sassari Cheremule

	scripts=("plot_winter.py" "plot_cape.py" "plot_convergence.py" "plot_gph_t_500.py" "plot_gph_t_850.py" \
		"plot_gph_thetae_850.py" "plot_hsnow.py" "plot_jetstream.py" "plot_pres_t2m_winds10m.py" "plot_rain_acc.py"\
		"plot_rain_clouds.py" "plot_winds10m.py" "plot_vorticity.py" "plot_soil_moisture.py" "plot_tmax.py" "plot_tmin.py" \
		"plot_sat.py" "plot_gph_500_mslp.py" "plot_t_850_anom.py" "plot_t2m_anom.py" "plot_z_500_anom.py" \
		"plot_hsnow_anom.py" "plot_tmax2m_anom.py" "plot_tmin2m_anom.py" "plot_mslp_anomaly.py" "plot_rain_acc_24.py")

	projections=("euratl" "it" "de")

	parallel -j 3 --delay 1 python ::: "${scripts[@]}" ::: "${projections[@]}"
	rm ${MODEL_DATA_FOLDER}*.py
fi

############################################################

# If we at least produced some netcdf and images we assume the run is succesfull and 
# store the run processed in a text file. Unfortunately there is no easy way to check
# if the processing is really succesfull as we have many errors in the process that
# don't undermine the whole processing! 

N_NETCDF_FILES=`find . -type f -name '*.nc' -printf x | wc -c`
N_IMAGES=`find . -type f -name '*.png' -printf x | wc -c`

if [ $N_NETCDF_FILES -ge 2 ] && [ $N_IMAGES -ge 10 ]; then
	echo ${latest_run} > last_processed_run.txt
fi


# SECTION 3 - IMAGES UPLOAD ############################################################
# Use ncftpbookmarks to add a new FTP server with credentials
if [ "$DATA_UPLOAD" = true ]; then
	echo "----------------------------------------------------------------------------------------------"
	echo "icon-forecasts: Starting FTP uploading - `date`"
	echo "----------------------------------------------------------------------------------------------"
	# First upload meteograms
	ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} icon_italy/meteograms meteogram_*
	#
	# Then upload the other pictures
	#
	images_output=("tmax_2m_anom" "tmin_2m_anom" "hsnow_anom" "gph_t_850" "gph_t_500" "gph_thetae_850" "precip_clouds" "winds10m" "winds_jet" "vort_850" "hsnow" "cape"\
		"conv" "soil_moisture" "t_v_pres" "precip_acc_24" "precip_acc" "vort_850" "winter" "tmax" "tmin" "sat" "gph_500_mslp" "t_850_anom" \
		"t_2m_anom" "z_500_anom" "mslp_anom")

	# suffix for naming
	projections_output=("" "it/" "de/")
	# remote folder on server
	projections_output_folder=("icon_forecasts" "icon_italy" "icon_de")

	# Create a lisf of all the images to upload 
	upload_elements=()
	for i in "${!projections_output[@]}"; do
		for j in "${images_output[@]}"; do
				upload_elements+=("${projections_output_folder[$i]}/${j} ./${projections_output[$i]}${j}_*")
		done
	done


	# for k in "${upload_elements[@]}"; do
	# 	ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} ${k}
	# done
	# Parallel version
    # Unfortunately I cannot find a way to make it work with GNU parallel
	num_procs=5
	num_iters=${#upload_elements[@]}
	num_jobs="\j"  # The prompt escape for number of jobs currently running
	for ((i=0; i<num_iters; i++)); do
		while (( ${num_jobs@P} >= num_procs )); do
		wait -n
		done
	ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} ${upload_elements[$i]} &
	done

fi 

# SECTION 4 - CLEANING ############################################################

echo "----------------------------------------------------------------------------------------------"
echo "icon-forecasts: Finished cleaning up - `date`"
echo "----------------------------------------------------------------------------------------------"

############################################################

cd -