diff --git a/calculation/parser.py b/calculation/parser.py index 4884410..c8c0d59 100644 --- a/calculation/parser.py +++ b/calculation/parser.py @@ -30,9 +30,8 @@ def parse_xml_to_object(file=OPTIONS_FILE): xml_root = xml_tree.getroot() start_date_time_str = xml_root.find('imin').text start_date_time = datetime.strptime(start_date_time_str, '%Y-%m-%d %H:%M:%S') - # TODO: check if end date is needed - # end_date_time_str = xml_root.find('imax').text - # end_date_time = datetime.strptime(end_date_time_str, '%Y-%m-%d %H:%M:%S') + end_date_time_str = xml_root.find('imax').text + end_date_time = datetime.strptime(end_date_time_str, '%Y-%m-%d %H:%M:%S') nx = (xml_root.find('nx').text) ny = (xml_root.find('ny').text) min_height = xml_root.find('minheight').text if xml_root.find( @@ -40,7 +39,7 @@ def parse_xml_to_object(file=OPTIONS_FILE): return { 'start_date_time': start_date_time, - # 'end_date_time': end_date_time, + 'end_date_time': end_date_time, 'out_longitude': xml_root.find('se_lon').text, 'out_latitude': xml_root.find('se_lat').text, 'num_x_grid': nx, @@ -111,7 +110,7 @@ def parse_csv_to_object(file=MEASUREMENTS_FILE): def parse_command_file(user_params): start_date_time = user_params['start_date_time'] - end_date_time = releases_params[-1]['end_date_time'] + timedelta(hours=1) + end_date_time = user_params['end_date_time'] + timedelta(hours=1) command_body = f"""&COMMAND LDIRECT= -1, ! Simulation direction in time ; 1 (forward) or -1 (backward) IBDATE= {start_date_time.strftime('%Y%m%d')}, ! Start date of the simulation ; YYYYMMDD: YYYY=year, MM=month, DD=day @@ -268,8 +267,8 @@ def parse_releases_file(releases_params): def process_releases(releases_params, user_params, start_calc_time): series_dirpath = f"/series/{user_params['series_id']}" - end_release_date = releases_params[-1]['end_date_time'] + timedelta(hours=1) - end_date_time_str = end_release_date.strftime('%Y%m%d%H%M%S') + end_date = user_params['end_date_time'] + timedelta(hours=1) + end_date_time_str = end_date.strftime('%Y%m%d%H%M%S') output_filename_prefix = f"grid_time_{end_date_time_str}" release_count = len(releases_params) @@ -283,7 +282,7 @@ def process_releases(releases_params, user_params, start_calc_time): parse_pathnames_file() # First date from user last is the last release date + 3 hours - download_data(user_params['start_date_time'], end_release_date) + download_data(user_params['start_date_time'], end_date) for i, param in enumerate(releases_params, start=1): id = param['id'] diff --git a/openstack/launch_instace.sh b/openstack/launch_instace.sh new file mode 100644 index 0000000..1322ce1 --- /dev/null +++ b/openstack/launch_instace.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="d1.xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}"; +TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH="/home/flexpart/.ssh/${VMNAME}.key" +calculation_dir=$(pwd) +done_file="$calculation_dir/done.txt" + +log_message() { + local message="$1" + local calc_log="$calculation_dir/calculations_server.log" + local vm_log="/home/flexpart/vm_launching.log" + + echo "$message" + echo -e "$TIME $message" | tee -a "$calc_log" >> "$vm_log" +} + +remove_vm() { + log_message "Preparing to remove the virtual machine $VMNAME ..." + + if [ -f $KEY_PATH ]; then + rm $KEY_PATH + log_message "Private key file '$KEY_PATH' has been removed." + else + log_message "Private key file '$KEY_PATH' not found." + fi + + # remove keypair if it exists + if openstack keypair show $VMNAME > /dev/null 2>&1; then + openstack keypair delete $VMNAME + log_message "Keypair ${VMNAME} has been deleted." + else + log_message "Keypair ${VMNAME} does not exist." + fi + + # check if it the instance exists + if openstack server show $VMNAME > /dev/null 2>&1; then + openstack server stop $VMNAME && openstack server delete $VMNAME + log_message "Instance ${VMNAME} has been deleted." + else + log_message "Instance ${VMNAME} does not exist, canceling." + fi +} + +# Define cleanup function for other errors +cleanup() { + local vm_created=$1 + local message=$2 + + if [[ $# -eq 1 && $1 =~ ^[0-9]+$ ]]; then + message="Command exited with status $1" + # Remove VM if it exists + elif [ "$vm_created" = true ]; then + remove_vm + fi + # Create a done.txt to indicate finishing the calculation + log_message "launch_error: $message\n" + touch "$done_file" + exit 1 +} + +test_quotas() { + # Get flavor information + flavor_info=$(openstack flavor show $FLAVOR) + + if [[ -z "$flavor_info" ]]; then + cleanup false "Flavor information is empty, can't start instance." + fi + + # Extract flavor cores and RAM + flavor_cores=$(echo "$flavor_info" | awk '/vcpus/ { print $4 }') + flavor_ram=$(echo "$flavor_info" | awk '/ram/ { print $4 }') + + # Get Nova limits + limits=$(nova limits 2>/dev/null | awk '/Cores|Instances|RAM/') + # Extract all cores, used cores, all instances, used instances, all RAM, and used RAM + read all_cores used_cores all_instances used_instances all_ram used_ram <<< $(echo $limits | awk '{ print $6, $4, $13, $11, $20, $18 }') + if (( $used_cores + $flavor_cores > $all_cores )); then + cleanup false "Core limit was reached! Fire $(expr $used_cores + $flavor_cores - $all_cores ) cores or change the flavor. Canceling ..." + fi + + if (( $used_instances + 1 >= $all_instances )); then + cleanup false "Instance limit was reached. Delete one of the instances. Canceling ..." + fi + + if (( $used_ram + $flavor_ram > $all_ram )); then + cleanup false "RAM limit was reached. Delete one of the instances or change the flavor. Canceling ..." + fi +} + +log_message "Process ID: $$" + +# Trap errors and call cleanup function, it will delete VM and create done.txt +trap 'cleanup true' ERR + +. /home/flexpart/.WRF-UNG # load openstack environment variables + +# Ensure calculation folder exists +if [[ ! -d "$calculation_dir" ]]; then + cleanup false "Calculation folder '$calculation_dir' does not exist." +fi + +# execute the test_quotas.sh script and provide the flavor name as an argument +if ! test_quotas; then + cleanup false "Quotas are exceeded, Canceling ..." +fi + +# create a series dir if not exist +xml_file="$calculation_dir/input/options.xml" +if [ ! -f "$xml_file" ]; then + cleanup false "File $xml_file does not exist." +fi + +series_id=$(grep -oP '\K[0-9]+' "$xml_file" | sed 's/^0*//') +series_path="/home/flexpart/series/$series_id" +mkdir -p "$series_path" + +# provide the calculation directory name to the VM +sed -i "6s@.*@DIR_NAME=$calculation_dir@" start_calculation.sh +log_message "Calculation path: $calculation_dir" +sed -i "7s@.*@SERIES_PATH=$series_path@" start_calculation.sh +log_message "Series path: $series_path" + +openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key" + +instance_id=$(nova boot --flavor $FLAVOR\ + --image f7eed42e-266d-4576-8ac6-b6dbbfa53233\ + --key-name $VMNAME\ + --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\ + --user-data /usr/local/bin/start_calculation.sh\ + $VMNAME\ + 2>/dev/null | awk '/ id / {print $4}') + +# Check if instance creation was successful +if [ -z "$instance_id" ]; then + cleanup true "Failed to create instance." +fi + +log_message "$TIME start creating VM $VMNAME, status - $STATUS." + +while true; do + STATUS=$(openstack server show --format value -c status $instance_id) + + if [ "$STATUS" == "ACTIVE" ]; then + IP=`openstack server show --format value -c addresses $instance_id | awk '{ split($1, v, "="); print v[2]}'` + + log_message "$TIME VM $VMNAME is $STATUS, IP address $IP" + log_message "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" + + # Main loop to check for done.txt creation + log_message "Calculation for '$calculation_dir' and series '$series_path' on VM $VMNAME started." + log_message "Waiting for 'done.txt' file in '$calculation_dir'..." + while [[ ! -f "$done_file" ]]; do + sleep "$TIMER" + done + log_message "File 'done.txt' detected! Removing the VM $VMNAME" + # TODO: add argument -test=true as a value from output to make it easy while testing + remove_vm + exit 0 + break + elif [ "$status" == "ERROR" ]; then + cleanup true "Instance $VMNAME creation failed." + fi + sleep 5 +done + +trap - ERR diff --git a/openstack/setup_instance.sh b/openstack/setup_instance.sh new file mode 100644 index 0000000..b58bdb1 --- /dev/null +++ b/openstack/setup_instance.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="m1.2xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}"; +TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH=.ssh/"${VMNAME}.key" + +openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key" + +nova boot --flavor $FLAVOR\ + --image cb3c7000-ed38-4e9d-a726-4d648dcbc9c9\ + --key-name $VMNAME\ + --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\ + $VMNAME + +echo -e "$TIME start creating VM $VMNAME, status - $STATUS\n" >> vm_launching.log + +for i in `seq 1 3`; do + echo -ne "$i attempt to start VM \033[0K\r" + sleep $TIMER & wait + + STATUS=`openstack server list | grep $VMNAME | awk '{ print $6 }'` + IP=`openstack server list | grep $VMNAME | awk '{ split($8, v, "="); print v[2]}'` + SYSTEM=`openstack server list | grep $VMNAME | awk '{ print $10 $11 }'` + + if [ "x$STATUS" = "xACTIVE" ]; then + printf "VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM\n" + echo "$TIME VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM" >> vm_launching.log + printf "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" + echo -e "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" >> vm_launching.log + exit 0 + fi +done + +if test -z "$STATUS"; then + echo "Launching $VMNAME failed" + echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log +else + printf "Launching $VMNAME failed with status: $STATUS" + echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log +fi + +private_key=".ssh/${VMNAME}.key" +if [ -f $private_key ]; then + rm $private_key + echo "Private key file '$private_key' has been removed." +else + echo "Private key file '$private_key' not found." +fi + +# remove keypair if it exists +if openstack keypair show $VMNAME > /dev/null 2>&1; then + openstack keypair delete $VMNAME + echo "Keypair ${VMNAME} has been deleted." +else + echo "Keypair ${VMNAME} does not exist" +fi + +# check if it the instance exists +if openstack server show $VMNAME > /dev/null 2>&1; then + openstack server stop $VMNAME && openstack server delete $VMNAME + echo "Instance ${VMNAME} has been deleted." +else + echo "Instance ${VMNAME} does not exist." +fi diff --git a/openstack/start_calculation.sh b/openstack/start_calculation.sh new file mode 100644 index 0000000..ca6f422 --- /dev/null +++ b/openstack/start_calculation.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# exit on the first error +set -e + +DIR_NAME=calculation_path +SERIES_PATH=series_path +NFS_SERVER=$(hostname -I) +uid=$(id username) +export LD_LIBRARY_PATH=/home/ubuntu/flexpart_lib/lib:$LD_LIBRARY_PATH + +# mount DIR_NAME folder to /data +sudo mount $NFS_SERVER:$DIR_NAME /data +sudo mount $NFS_SERVER:$SERIES_PATH /series +# could be removed in production +sudo mount $NFS_SERVER:/home/flexpart/series/grid_data /grid_data + +echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log + +sudo chown -R ubuntu /data/ /series /grid_data + +su -c "cd /home/ubuntu/calculation && python3.6 parser.py" ubuntu + +echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log + +# Change ownership recursively to the specified user +sudo chown -R "$uid:$uid" /data /series /grid_data +sudo umount /data /series /grid_data + +exit 0 diff --git a/simflex_v1/src/read_tintegr_srs.for b/simflex_v1/src/read_tintegr_srs.for index 261c52a..a82f6d9 100644 --- a/simflex_v1/src/read_tintegr_srs.for +++ b/simflex_v1/src/read_tintegr_srs.for @@ -109,9 +109,7 @@ srsred=REAL(loutstep)*AllSRS(i)%srstsum(:,:,1) call fname_srs(fsrspattern,id_obs(i),fsrsname,200) curlen=len_trim(fsrsname) - series_dirpath = '/series/'// - & trim(adjustl(int2str(series_id)))// - & '/'//trim(nuclide_name)//'/' + series_dirpath = '/series/'//trim(nuclide_name)//'/' open(1022,FILE=series_dirpath//fsrsname(1:curlen), & FORM='UNFORMATTED') write(1022)nlon,nlat