Openstack v1

calculation/parser.py

@@ -30,17 +30,16 @@ def parse_xml_to_object(file=OPTIONS_FILE):
   xml_root = xml_tree.getroot()
   start_date_time_str = xml_root.find('imin').text
   start_date_time = datetime.strptime(start_date_time_str, '%Y-%m-%d %H:%M:%S')
-  # TODO: check if end date is needed
-  # end_date_time_str = xml_root.find('imax').text
-  # end_date_time = datetime.strptime(end_date_time_str, '%Y-%m-%d %H:%M:%S')
+  end_date_time_str = xml_root.find('imax').text
+  end_date_time = datetime.strptime(end_date_time_str, '%Y-%m-%d %H:%M:%S')
   nx = (xml_root.find('nx').text)
   ny = (xml_root.find('ny').text)
   min_height = xml_root.find('minheight').text if xml_root.find(
       'minheight').text != '0' else '1'
 
   return {
       'start_date_time': start_date_time,
-      # 'end_date_time': end_date_time,
+      'end_date_time': end_date_time,
       'out_longitude': xml_root.find('se_lon').text,
       'out_latitude': xml_root.find('se_lat').text,
       'num_x_grid': nx,
@@ -111,7 +110,7 @@ def parse_csv_to_object(file=MEASUREMENTS_FILE):
 
 def parse_command_file(user_params):
   start_date_time = user_params['start_date_time']
-  end_date_time = releases_params[-1]['end_date_time'] + timedelta(hours=1)
+  end_date_time = user_params['end_date_time'] + timedelta(hours=1)
   command_body = f"""&COMMAND
  LDIRECT=              -1, ! Simulation direction in time   ; 1 (forward) or -1 (backward)
  IBDATE=         {start_date_time.strftime('%Y%m%d')}, ! Start date of the simulation   ; YYYYMMDD: YYYY=year, MM=month, DD=day
@@ -268,8 +267,8 @@ def parse_releases_file(releases_params):
 
 def process_releases(releases_params, user_params, start_calc_time):
   series_dirpath = f"/series/{user_params['series_id']}"
-  end_release_date = releases_params[-1]['end_date_time'] + timedelta(hours=1)
-  end_date_time_str = end_release_date.strftime('%Y%m%d%H%M%S')
+  end_date = user_params['end_date_time'] + timedelta(hours=1)
+  end_date_time_str = end_date.strftime('%Y%m%d%H%M%S')
   output_filename_prefix = f"grid_time_{end_date_time_str}"
   release_count = len(releases_params)
 
@@ -283,7 +282,7 @@ def process_releases(releases_params, user_params, start_calc_time):
   parse_pathnames_file()
 
   # First date from user last is the last release date + 3 hours
-  download_data(user_params['start_date_time'], end_release_date)
+  download_data(user_params['start_date_time'], end_date)
 
   for i, param in enumerate(releases_params, start=1):
     id = param['id']

openstack/launch_instace.sh

@@ -0,0 +1,166 @@
+#!/bin/bash
+
+HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="d1.xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}";
+TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH="/home/flexpart/.ssh/${VMNAME}.key"
+calculation_dir=$(pwd)
+done_file="$calculation_dir/done.txt"
+
+log_message() {
+  local message="$1"
+  local calc_log="$calculation_dir/calculations_server.log"
+  local vm_log="/home/flexpart/vm_launching.log"
+
+  echo "$message"
+  echo -e "$TIME $message" | tee -a "$calc_log" >> "$vm_log"
+}
+
+remove_vm() {
+  log_message "Preparing to remove the virtual machine $VMNAME ..."
+
+  if [ -f $KEY_PATH ]; then
+    rm $KEY_PATH
+    log_message "Private key file '$KEY_PATH' has been removed."
+  else
+    log_message "Private key file '$KEY_PATH' not found."
+  fi
+
+  # remove keypair if it exists
+  if openstack keypair show $VMNAME > /dev/null 2>&1; then
+    openstack keypair delete $VMNAME
+    log_message "Keypair ${VMNAME} has been deleted."
+  else
+    log_message "Keypair ${VMNAME} does not exist."
+  fi
+
+  # check if it the instance exists
+  if openstack server show $VMNAME > /dev/null 2>&1; then
+    openstack server stop $VMNAME && openstack server delete $VMNAME
+    log_message "Instance ${VMNAME} has been deleted."
+  else
+    log_message "Instance ${VMNAME} does not exist, canceling."
+  fi
+}
+
+# Define cleanup function for other errors
+cleanup() {
+    local vm_created=$1
+    local message=$2
+
+    if [[ $# -eq 1 && $1 =~ ^[0-9]+$ ]]; then
+      message="Command exited with status $1"
+    # Remove VM if it exists
+    elif [ "$vm_created" = true ]; then
+      remove_vm
+    fi
+    # Create a done.txt to indicate finishing the calculation
+    log_message "launch_error: $message\n"
+    touch "$done_file"
+    exit 1
+}
+
+test_quotas() {
+  # Get flavor information
+  flavor_info=$(openstack flavor show $FLAVOR)
+
+  if [[ -z "$flavor_info" ]]; then
+    cleanup false "Flavor information is empty, can't start instance."
+  fi
+
+  # Extract flavor cores and RAM
+  flavor_cores=$(echo "$flavor_info" | awk '/vcpus/ { print $4 }')
+  flavor_ram=$(echo "$flavor_info" | awk '/ram/ { print $4 }')
+
+  # Get Nova limits
+  limits=$(nova limits 2>/dev/null | awk '/Cores|Instances|RAM/')
+  # Extract all cores, used cores, all instances, used instances, all RAM, and used RAM
+  read all_cores used_cores all_instances used_instances all_ram used_ram <<< $(echo $limits | awk '{ print $6, $4, $13, $11, $20, $18 }')
+  if (( $used_cores + $flavor_cores > $all_cores )); then
+	  cleanup false "Core limit was reached! Fire $(expr $used_cores + $flavor_cores - $all_cores ) cores or change the flavor. Canceling ..."
+  fi
+
+  if (( $used_instances + 1 >= $all_instances )); then
+    cleanup false "Instance limit was reached. Delete one of the instances. Canceling ..."
+  fi
+
+  if (( $used_ram + $flavor_ram > $all_ram )); then
+    cleanup false "RAM limit was reached. Delete one of the instances or change the flavor. Canceling ..."
+  fi
+}
+
+log_message "Process ID: $$"
+
+# Trap errors and call cleanup function, it will delete VM and create done.txt
+trap 'cleanup true' ERR
+
+. /home/flexpart/.WRF-UNG # load openstack environment variables
+
+# Ensure calculation folder exists
+if [[ ! -d "$calculation_dir" ]]; then
+    cleanup false "Calculation folder '$calculation_dir' does not exist."
+fi
+
+# execute the test_quotas.sh script and provide the flavor name as an argument
+if ! test_quotas; then
+  cleanup false "Quotas are exceeded, Canceling ..."
+fi
+
+# create a series dir if not exist
+xml_file="$calculation_dir/input/options.xml"
+if [ ! -f "$xml_file" ]; then
+    cleanup false "File $xml_file does not exist."
+fi
+
+series_id=$(grep -oP '<id_series>\K[0-9]+' "$xml_file" | sed 's/^0*//')
+series_path="/home/flexpart/series/$series_id"
+mkdir -p "$series_path"
+
+# provide the calculation directory name to the VM
+sed -i "6s@.*@DIR_NAME=$calculation_dir@" start_calculation.sh
+log_message "Calculation path: $calculation_dir"
+sed -i "7s@.*@SERIES_PATH=$series_path@" start_calculation.sh
+log_message "Series path: $series_path"
+
+openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key"
+
+instance_id=$(nova boot --flavor $FLAVOR\
+        --image f7eed42e-266d-4576-8ac6-b6dbbfa53233\
+        --key-name $VMNAME\
+        --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\
+        --user-data /usr/local/bin/start_calculation.sh\
+        $VMNAME\
+        2>/dev/null | awk '/ id / {print $4}')
+
+# Check if instance creation was successful
+if [ -z "$instance_id" ]; then
+    cleanup true "Failed to create instance."
+fi
+
+log_message "$TIME start creating VM $VMNAME, status - $STATUS."
+
+while true; do
+  	STATUS=$(openstack server show --format value -c status $instance_id)
+
+    if [ "$STATUS" == "ACTIVE" ]; then
+     	IP=`openstack server show --format value -c addresses $instance_id | awk '{ split($1, v, "="); print v[2]}'`
+
+      log_message "$TIME VM $VMNAME is $STATUS, IP address $IP"
+      log_message "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n"
+
+        # Main loop to check for done.txt creation
+        log_message "Calculation for '$calculation_dir' and series '$series_path' on VM $VMNAME started."
+        log_message "Waiting for 'done.txt' file in '$calculation_dir'..."
+        while [[ ! -f "$done_file" ]]; do
+          sleep "$TIMER"
+        done
+        log_message "File 'done.txt' detected! Removing the VM $VMNAME"
+        # TODO: add argument -test=true as a value from output to make it easy while testing
+        remove_vm
+        exit 0
+        break
+    elif [ "$status" == "ERROR" ]; then
+        cleanup true "Instance $VMNAME creation failed."
+    fi
+    sleep 5
+done
+
+trap - ERR

openstack/setup_instance.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="m1.2xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}";
+TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH=.ssh/"${VMNAME}.key"
+
+openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key"
+
+nova boot --flavor $FLAVOR\
+        --image cb3c7000-ed38-4e9d-a726-4d648dcbc9c9\
+        --key-name $VMNAME\
+        --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\
+        $VMNAME
+
+echo -e "$TIME start creating VM $VMNAME, status - $STATUS\n" >> vm_launching.log
+
+for i in `seq 1 3`; do
+        echo -ne "$i attempt to start VM \033[0K\r"
+	sleep $TIMER & wait
+
+  	STATUS=`openstack server list | grep $VMNAME | awk '{ print $6 }'`
+  	IP=`openstack server list | grep $VMNAME | awk '{ split($8, v, "="); print v[2]}'`
+  	SYSTEM=`openstack server list | grep $VMNAME | awk '{ print $10 $11 }'`
+
+  	if [ "x$STATUS" = "xACTIVE" ]; then
+		printf "VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM\n"
+    		echo "$TIME VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM" >> vm_launching.log
+		printf "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n"
+		echo -e "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" >> vm_launching.log
+		exit 0
+	fi
+done
+
+if test -z "$STATUS"; then
+	echo "Launching $VMNAME failed"
+	echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log
+else
+	printf "Launching $VMNAME failed with status: $STATUS"
+	echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log
+fi
+
+private_key=".ssh/${VMNAME}.key"
+if [ -f $private_key ]; then
+	rm $private_key
+        echo "Private key file '$private_key' has been removed."
+else
+	echo "Private key file '$private_key' not found."
+fi
+
+# remove keypair if it exists
+if openstack keypair show $VMNAME > /dev/null 2>&1; then
+  openstack keypair delete $VMNAME
+  echo "Keypair ${VMNAME} has been deleted."
+else
+  echo "Keypair ${VMNAME} does not exist"
+fi
+
+# check if it the instance exists
+if openstack server show $VMNAME > /dev/null 2>&1; then
+  openstack server stop $VMNAME && openstack server delete $VMNAME
+  echo "Instance ${VMNAME} has been deleted."
+else
+  echo "Instance ${VMNAME} does not exist."
+fi

openstack/start_calculation.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# exit on the first error
+set -e
+
+DIR_NAME=calculation_path
+SERIES_PATH=series_path
+NFS_SERVER=$(hostname -I)
+uid=$(id username)
+export LD_LIBRARY_PATH=/home/ubuntu/flexpart_lib/lib:$LD_LIBRARY_PATH
+
+# mount DIR_NAME folder to /data
+sudo mount $NFS_SERVER:$DIR_NAME /data
+sudo mount $NFS_SERVER:$SERIES_PATH /series
+# could be removed in production
+sudo mount $NFS_SERVER:/home/flexpart/series/grid_data /grid_data
+
+echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log
+
+sudo chown -R ubuntu /data/ /series /grid_data
+
+su -c "cd /home/ubuntu/calculation && python3.6 parser.py" ubuntu
+
+echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log
+
+# Change ownership recursively to the specified user
+sudo chown -R "$uid:$uid" /data /series /grid_data
+sudo umount /data /series /grid_data
+
+exit 0

simflex_v1/src/read_tintegr_srs.for

@@ -109,9 +109,7 @@
             srsred=REAL(loutstep)*AllSRS(i)%srstsum(:,:,1)
              call fname_srs(fsrspattern,id_obs(i),fsrsname,200)
              curlen=len_trim(fsrsname)
-             series_dirpath = '/series/'//
-     &                        trim(adjustl(int2str(series_id)))//
-     &                        '/'//trim(nuclide_name)//'/'
+             series_dirpath = '/series/'//trim(nuclide_name)//'/'
               open(1022,FILE=series_dirpath//fsrsname(1:curlen),
      &                  FORM='UNFORMATTED')
               write(1022)nlon,nlat