Post a comment about sporadic failure

When a retry job passed, then the original failure was likely
just a sporadic issue. We now post a comment to the original job
with this information.

Issue: https://progress.opensuse.org/issues/98862

Still todo: enable _TRIGGER_JOB_DONE_HOOK=1 to allow processing
passed investigation jobs.

Author: perlpunk

openqa-investigate

@@ -180,6 +180,32 @@ $comment_text"
     fi
 }
 
+post-investigate() {
+    local id=$1 old_name=$2
+    local rc=0 status
+    [[ ! "$old_name" =~ investigate:retry$ ]] && echo "Job is ':investigate:' already, skipping investigation" && return 0
+    result="$(echo "$job_data" | runjq -r '.job.result')" || return $?
+    investigate_origin="$(echo "$job_data" | runjq -r '.job.settings.OPENQA_INVESTIGATE_ORIGIN')" || return 1
+    origin_job_id=${investigate_origin#"$host_url/t"}
+    comment="Investigate retry job: $host_url/t$id"
+    if [[ "$result" == passed ]]; then
+        comment+=" passed, likely a sporadic failure"
+    else
+        comment+=" failed, likely not a sporadic failure"
+    fi
+
+    # meanwhile the original job might have been deleted already, handle
+    # gracefully
+    out=$("${client_call[@]}" -X POST jobs/"$origin_job_id"/comments text="$comment" 2>&1) || rc=$?
+    if [[ "$rc" -ne 0 ]]; then
+        status=$(echo "$out" | runjq .error_status 2>/dev/null || echo "unknown")
+        if [[ "$status" != "404" ]]; then
+            echoerr "Unexpected error encountered when posting comments on job $origin_job_id after investigation job $id failed: '$out'"
+            return 2
+        fi
+    fi
+}
+
 # crosscheck
 # 1. current job/build + current test -> check if reproducible/sporadic
 # 2. current job/build + last good test (+ last good needles) -> check for
@@ -196,17 +222,7 @@ investigate() {
     [[ $? != 0 ]] && echoerr "unable to query job data for $id: $job_data" && return 1
     old_name="$(echo "$job_data" | runjq -r '.job.test')" || return $?
     if [[ "$old_name" =~ ":investigate:" ]]; then
-        [[ ! "$old_name" =~ investigate:retry$ ]] && echo "Job is ':investigate:' already, skipping investigation" && return 0
-        investigate_origin="$(echo "$job_data" | runjq -r '.job.settings.OPENQA_INVESTIGATE_ORIGIN')" || return 1
-        origin_job_id=${investigate_origin#"$host_url/t"}
-        # meanwhile the original job might have been deleted already, handle
-        # gracefully
-        out=$("${client_call[@]}" -X POST jobs/"$origin_job_id"/comments text="Investigate retry job: $host_url/t$id failed, likely not a sporadic failure" 2>&1) || \
-            status=$(echo "$out" | runjq .error_status 2>/dev/null || echo "unknown")
-            if [[ "$status" != "404" ]]; then
-                echoerr "Unexpected error encountered when posting comments on job $origin_job_id after investigation job $id failed: '$out'"
-                return 2
-            fi
+        post-investigate "$id" "$old_name" || return $?
         return 0
     fi
     clone="$(echo "$job_data" | runjq -r '.job.clone_id')" || return $?

test/02-investigate.t

@@ -3,7 +3,7 @@
 source test/init
 bpan:source bashplus +err +fs +sym
 
-plan tests 31
+plan tests 35
 
 host=localhost
 url=https://localhost
@@ -37,6 +37,8 @@ openqa-cli() {
         echo '{"job": { "test": "vim", "test": "vim:investigate:abc", "result": "failed" } }'
     elif [[ "$1 $2" == "--json jobs/34" ]]; then
         echo '{"job": { "test": "vim:investigate:retry", "result": "failed", "settings": {"OPENQA_INVESTIGATE_ORIGIN": "35"} } }'
+    elif [[ "$1 $2" == "--json jobs/35" ]]; then
+        echo '{"job": { "test": "vim:investigate:retry", "result": "passed", "settings": {"OPENQA_INVESTIGATE_ORIGIN": "35"} } }'
     elif [[ $@ == "-X POST jobs/30/comments text=Starting investigation for job 31" ]]; then
         echo '{"id": 1234}'
     elif [[ $@ == $'-X PUT jobs/30/comments/1234 text=Automatic investigation jobs for job 31:\n\nfoo' ]]; then
@@ -105,11 +107,17 @@ has "$got" 'Job w/o job group, $exclude_no_group is set, skipping investigation'
 
 try investigate 33
 is "$rc" 0 'success (33)'
-has "$got" "Job is ':investigate:' already, skipping investigation" "skip :investigate:"
+has "$got" "Job is ':investigate:' already, skipping investigation" "skip :investigate: (33)"
 
 try investigate 34
 is "$rc" 2 'mocked function returned failure (34)'
-has "$got" "Commenting 35" "Posting comment on OPENQA_INVESTIGATE_ORIGIN"
+has "$got" "Commenting 35" "Posting comment on OPENQA_INVESTIGATE_ORIGIN (34)"
+has "$got" "likely not a sporadic" "not sporadic (34)"
+
+try investigate 35
+is "$rc" 2 'mocked function returned failure (35)'
+has "$got" "Commenting 35" "Posting comment on OPENQA_INVESTIGATE_ORIGIN (35)"
+has "$got" "likely a sporadic" "sporadic (35)"
 
 # test syncing via investigation comment; we're first
 try force=true sync_via_investigation_comment 31 30