Add Email Notification System for scrontab-Launched Rocoto Workflows

dev/workflow/generate_workflows.sh

@@ -82,6 +82,7 @@ EOF
 }
 
 set -eu
+shopt -s inherit_errexit
 
 # Set default options
 HOMEgfs=""
@@ -187,6 +188,11 @@ function send_email() {
     echo "${_body}" | mail -s "${_subject}" "${_email}"
 }
 
+# Export MAILTO if email was provided via -e flag and is not empty
+if [[ "${_set_email}" == "true" && -n "${_email}" ]]; then
+    export MAILTO="${_email}"
+fi
+
 function delete_dir() {
     local dir_to_rm="${1:-}"
     if [[ -z "${dir_to_rm}" ]]; then
@@ -531,9 +537,9 @@ for _case in "${_yaml_list[@]}"; do
     _pslot="${_case}${_tag}"
     _create_exp_cmd="./create_experiment.py -y ${_yaml_dir}/${_case}.yaml --overwrite"
     if [[ "${_verbose}" == true ]]; then
-        pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd}
+        pslot=${_pslot} RUNTESTS=${_runtests} MAILTO="${MAILTO:-}" ${_create_exp_cmd}
     else
-        if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then
+        if ! pslot=${_pslot} RUNTESTS=${_runtests} MAILTO="${MAILTO:-}" ${_create_exp_cmd} 2> stderr 1> stdout; then
             _output=$(cat stdout stderr)
             _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status.  Output:"
             _message="${_message}"$'\n'"${_output}"
@@ -584,43 +590,82 @@ for _case in "${_yaml_list[@]}"; do
 
     if [[ "${_use_scron}" == true ]]; then
         {
-            grep "^#.*${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab"
+            # Skip MAILTO lines (will be added once at the top later)
+            grep "^####" "${cron_file}" | grep -v "MAILTO="
             grep "^#SCRON" "${cron_file}"
-            grep "${scron_sh_file}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab"
+            grep "${scron_sh_file}" "${cron_file}"
         } >> tests.cron
     else
         grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron
     fi
 done
 echo
 
+# Function to format crontab comment lines with consistent width
+_format_crontab_comment_line() {
+    set -e
+    local text="${1:-}"
+    local total_length=65
+
+    if [[ -z "${text}" ]]; then
+        printf '%*s' "${total_length}" '' | tr ' ' '#'
+    else
+        local text_with_spaces=" ${text} "
+        local text_len=${#text_with_spaces}
+        local remaining=$((total_length - text_len))
+        local left_padding=$((remaining / 2))
+        local right_padding=$((remaining - left_padding))
+        printf '%*s%s%*s' "${left_padding}" '' "${text_with_spaces}" "${right_padding}" '' | tr ' ' '#'
+    fi
+    echo
+}
+
+# Add email to tests.cron if provided via -e flag
+if [[ "${_set_email}" == "true" ]]; then
+    # Remove any existing MAILTO lines from tests.cron
+    sed -i '/MAILTO=/d' tests.cron 2> /dev/null || true
+
+    # Add MAILTO as the first line - format depends on whether using scrontab
+    if [[ "${_use_scron}" == true ]]; then
+        # For scrontab, use formatted comment line
+        mailto_formatted=$(_format_crontab_comment_line "MAILTO=${_email}")
+        sed -i "1i ${mailto_formatted}" tests.cron
+    else
+        # For regular crontab, use plain directive
+        sed -i "1i MAILTO=${_email}" tests.cron
+    fi
+fi
+
 # Update the cron
 if [[ "${_update_cron}" == "true" ]]; then
     printf "Updating the existing crontab\n\n"
-    echo
     rm -f existing.cron final.cron "${_verbose_flag}"
     touch existing.cron final.cron
-
+    echo
     ${_crontab_cmd} -l | grep -v "no crontab for" > existing.cron || true
 
+    # Show warning only if MAILTO is not set as env variable and not present in existing.cron
+    if [[ "${_set_email}" == "false" && -z "${MAILTO:-}" ]]; then
+        if ! grep -q "MAILTO=" existing.cron 2> /dev/null; then
+            echo -e "\033[0;33mWARNING:\033[0m Set \033[0;32mexport MAILTO=\"your_email\"\033[0m in your .bashrc or use generate_workflows.sh with \033[0;32m-e \"your_email\"\033[0m to receive job failure notifications."
+        fi
+    fi
+
     if [[ "${_debug}" == "true" ]]; then
         echo "Existing crontab: "
         echo "#######################"
         cat existing.cron
         echo "#######################"
     fi
 
-    if [[ "${_set_email}" == "true" ]]; then
-        # Replace the existing email in the crontab
-        if [[ "${_verbose}" == "true" ]]; then
-            printf "Updating crontab/scrontab email to %s\n\n" "${_email}"
-        fi
+    # Remove MAILTO lines from both existing.cron and tests.cron to prevent duplicates
+    sed -i '/MAILTO=/d' existing.cron 2> /dev/null || true
+    sed -i '/MAILTO=/d' tests.cron 2> /dev/null || true
 
-        if [[ "${_use_scron}" == true ]]; then
-            sed -i "s/.*--mail-user.*/#SCRON --mail-user=\"${_email}\"/" tests.cron
-        else
-            sed -i "s/^MAILTO.*/MAILTO=\"${_email}\"/" existing.cron
-        fi
+    # Extract MAILTO line from the crontab file and put at top of final.cron
+    mailto_line=$(grep "MAILTO=" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" 2> /dev/null | head -1 || echo "")
+    if [[ -n "${mailto_line}" ]]; then
+        echo "${mailto_line}" > final.cron
     fi
 
     cat existing.cron tests.cron >> final.cron
@@ -634,10 +679,17 @@ if [[ "${_update_cron}" == "true" ]]; then
 
     ${_crontab_cmd} final.cron
 else
+    # Show warning only if MAILTO is not set as env variable and not present in tests.cron
+    if [[ "${_set_email}" == "false" && -z "${MAILTO:-}" ]]; then
+        if ! grep -q "MAILTO=" tests.cron 2> /dev/null; then
+            echo -e "\033[0;33mWARNING:\033[0m Set \033[0;32mexport MAILTO=\"your_email\"\033[0m in your .bashrc or use generate_workflows.sh with \033[0;32m-e \"your_email\"\033[0m to receive job failure notifications."
+        fi
+    fi
     _message="Add the following to your crontab or scrontab to start running:"
     _cron_tests=$(cat tests.cron)
     _message="${_message}"$'\n'"${_cron_tests}"
     echo "${_message}"
+    echo
     if [[ "${_set_email}" == true ]]; then
         final_message="${final_message:-}"$'\n'"${_message}"
     fi

dev/workflow/rocoto/rocoto_scron.sh.j2

@@ -0,0 +1,73 @@
+#! /usr/bin/env bash
+source "{{ HOMEgfs }}/dev/ush/gw_setup.sh"
+
+# Run rocotorun
+bash -c "{{ rocotorunstr }}"
+
+# Monitor for failed jobs using rocotostat
+LOCKFILE="{{ expdir }}/.failed_jobs.lock"
+ROCOTOSTAT=$(command -v rocotostat)
+
+if [[ -n "${ROCOTOSTAT}" ]]; then
+    FAILED_JOBS=$(${ROCOTOSTAT} -d "{{ expdir }}/{{ pslot }}.db" -w "{{ expdir }}/{{ pslot }}.xml" -c all 2> /dev/null | grep -E 'DEAD')
+
+    if [[ -n "${FAILED_JOBS}" ]]; then
+        # Read previously reported failures
+        PREV_FAILED=""
+        if [[ -f "${LOCKFILE}" ]]; then
+            PREV_FAILED=$(cat "${LOCKFILE}")
+        fi
+
+        # Check for NEW failures only (not just changes)
+        NEW_FAILURES=""
+        while IFS= read -r job; do
+            if [[ -n "${job}" ]] && ! echo "${PREV_FAILED}" | grep -qF "${job}"; then
+                NEW_FAILURES="${NEW_FAILURES}${job}"$'\n'
+            fi
+        done <<< "${FAILED_JOBS}"
+
+        # Send email only if there are NEW failures
+        if [[ -n "${NEW_FAILURES}" ]]; then
+            MSGFILE="/tmp/rocoto_fail_msg_$$.txt"
+            {
+                echo "The following jobs have failed in experiment {{ pslot }} on ${MACHINE_ID}:"
+                echo ""
+
+                # Format each failed job with detailed information
+                while IFS= read -r line; do
+                    if [[ -n "${line}" ]]; then
+                        # Parse rocotostat output: Cycle Task JobID State Try MaxTries Duration
+                        read -r cycle task jobid state try maxtries duration <<< "${line}"
+                        # Extract YYYYMMDDHH from cycle (first 10 characters)
+                        cycle_short=${cycle:0:10}
+                        # Get current timestamp
+                        timestamp=$(date -u '+%m/%d/%y %H:%M:%S UTC')
+
+                        # Format similar to user's example
+                        echo "${timestamp} :: {{ pslot }}.xml :: Cycle ${cycle}, Task ${task}, jobid=${jobid}, in state ${state}, ran for ${duration} seconds, try=${try} (of ${maxtries})"
+                        echo "Check log: {{ comroot }}/{{ pslot }}/logs/${cycle_short}/${task}.log"
+                        echo ""
+                    fi
+                done <<< "${NEW_FAILURES}"
+            } > "${MSGFILE}"
+
+            # Try to send email
+            EMAIL="{{ mailto }}"
+            hostname_domain=$(hostname -d)
+            FROM_EMAIL="no-reply@${hostname_domain}"
+            if [[ "${EMAIL}" != "None" ]] && command -v mail &> /dev/null; then
+                # On Gaea, the mail utility requires the -v (verbose) flag to ensure delivery.
+                # To avoid receiving verbose output as an actual email, a spoofed 'from' address is used for notifications.
+                mail -r "${FROM_EMAIL}" -v -s "[{{ pslot }}] Workflow Job Failures Detected" "${EMAIL}" < "${MSGFILE}" 2>&1
+            fi
+
+            rm -f "${MSGFILE}"
+        fi
+
+        # Always update lockfile to reflect current failures
+        echo "${FAILED_JOBS}" > "${LOCKFILE}"
+    else
+        # No failures, remove lockfile if it exists
+        [[ -f "${LOCKFILE}" ]] && rm -f "${LOCKFILE}"
+    fi
+fi