Merge pull request #204 from perlpunk/post-investigate

Post comment about sporadic failure

Author: mergify[bot]

_common

@@ -75,6 +75,11 @@ runcurl() {
     echo "$body"
 }
 
+openqa-api-get() {
+    client_args=(api --header 'User-Agent: openqa-investigate (https://github.com/os-autoinst/scripts)' --host "$host_url")
+    openqa-cli "${client_args[@]}" --json "$@"
+}
+
 comment_on_job() {
     local id=$1 comment=$2 force_result=${3:-''}
     if $enable_force_result && [[ -n $force_result ]]; then

openqa-investigate

@@ -180,6 +180,32 @@ $comment_text"
     fi
 }
 
+post-investigate() {
+    local id=$1 old_name=$2
+    local rc=0 status
+    [[ ! "$old_name" =~ investigate:retry$ ]] && echo "Job is ':investigate:' already, skipping investigation" && return 0
+    result="$(echo "$job_data" | runjq -r '.job.result')" || return $?
+    investigate_origin="$(echo "$job_data" | runjq -r '.job.settings.OPENQA_INVESTIGATE_ORIGIN')" || return 1
+    origin_job_id=${investigate_origin#"$host_url/t"}
+    comment="Investigate retry job: $host_url/t$id"
+    if [[ "$result" == passed ]]; then
+        comment+=" passed, likely a sporadic failure"
+    else
+        comment+=" failed, likely not a sporadic failure"
+    fi
+
+    # meanwhile the original job might have been deleted already, handle
+    # gracefully
+    out=$("${client_call[@]}" -X POST jobs/"$origin_job_id"/comments text="$comment" 2>&1) || rc=$?
+    if [[ "$rc" -ne 0 ]]; then
+        status=$(echo "$out" | runjq .error_status 2>/dev/null || echo "unknown")
+        if [[ "$status" != "404" ]]; then
+            echoerr "Unexpected error encountered when posting comments on job $origin_job_id after investigation job $id failed: '$out'"
+            return 2
+        fi
+    fi
+}
+
 # crosscheck
 # 1. current job/build + current test -> check if reproducible/sporadic
 # 2. current job/build + last good test (+ last good needles) -> check for
@@ -196,16 +222,7 @@ investigate() {
     [[ $? != 0 ]] && echoerr "unable to query job data for $id: $job_data" && return 1
     old_name="$(echo "$job_data" | runjq -r '.job.test')" || return $?
     if [[ "$old_name" =~ ":investigate:" ]]; then
-        [[ ! "$old_name" =~ investigate:retry$ ]] && echo "Job is ':investigate:' already, skipping investigation" && return 0
-        investigate_origin="$(echo "$job_data" | runjq -r '.job.settings.OPENQA_INVESTIGATE_ORIGIN')" || return 1
-        origin_job_id=${investigate_origin#"$host_url/t"}
-        # meanwhile the original job might have been deleted already, handle
-        # gracefully
-        out=$("${client_call[@]}" -X POST jobs/"$origin_job_id"/comments text="Investigate retry job: $host_url/t$id failed, likely not a sporadic failure" 2>/dev/null) || \
-            if [[ $(echo "$out" | runjq .error_status) != "404" ]]; then
-                echoerr "Unexpected error encountered when posting comments on job $origin_job_id after investigation job $id failed: '$out'"
-                return 2
-            fi
+        post-investigate "$id" "$old_name" || return $?
         return 0
     fi
     clone="$(echo "$job_data" | runjq -r '.job.clone_id')" || return $?
@@ -241,6 +258,7 @@ investigate() {
 }
 
 main() {
+    local id=${1:?"Need 'job_id'"}
     client_prefix=''
     [ "$dry_run" = "1" ] && client_prefix="echo"
     set +u

openqa-label-known-issues-and-investigate-hook

@@ -8,6 +8,9 @@ set -eo pipefail
 dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 PATH=$dir:$PATH
 
+# shellcheck source=/dev/null
+source "$dir"/_common
+
 host="${host:-"openqa.opensuse.org"}"
 scheme="${scheme:-"https"}"
 host_url="$scheme://$host"
@@ -21,9 +24,23 @@ investigate-and-bisect() {
     return "$rc"
 }
 
+label() {
+    local url=$1
+    openqa-label-known-issues "$url" | sed -n 's/\[\([^]]*\)\].*Unknown issue, to be reviewed.*/\1/p'
+}
+
 hook() {
     local id="${1:?"Need 'job_id'"}"
-    openqa-label-known-issues "$host_url/tests/$id" | sed -n 's/\[\([^]]*\)\].*Unknown issue, to be reviewed.*/\1/p' | investigate-and-bisect
+    local url=$host_url/tests/$id
+    job_data=$(openqa-api-get "jobs/$id")
+    state="$(echo "$job_data" | runjq -r '.job.state')" || return $?
+    result="$(echo "$job_data" | runjq -r '.job.result')" || return $?
+    [[ "$state" != "done" ]] && return
+    if [[ "$result" != passed ]]; then
+        label "$url" | investigate-and-bisect
+    else
+        echo "$url" | investigate-and-bisect
+    fi
 }
 
 caller 0 >/dev/null || hook "$@"

test/02-investigate.t

@@ -3,7 +3,7 @@
 source test/init
 bpan:source bashplus +err +fs +sym
 
-plan tests 27
+plan tests 35
 
 host=localhost
 url=https://localhost
@@ -33,6 +33,12 @@ openqa-cli() {
         echo '{"job": { "test": "vim", "priority": 50, "settings" : {} } }'
     elif [[ "$1 $2" == "--json jobs/27" ]]; then
         echo '{"job": { "test": "vim", "clone_id" : 28 } }'
+    elif [[ "$1 $2" == "--json jobs/33" ]]; then
+        echo '{"job": { "test": "vim", "test": "vim:investigate:abc", "result": "failed" } }'
+    elif [[ "$1 $2" == "--json jobs/34" ]]; then
+        echo '{"job": { "test": "vim:investigate:retry", "result": "failed", "settings": {"OPENQA_INVESTIGATE_ORIGIN": "35"} } }'
+    elif [[ "$1 $2" == "--json jobs/35" ]]; then
+        echo '{"job": { "test": "vim:investigate:retry", "result": "passed", "settings": {"OPENQA_INVESTIGATE_ORIGIN": "35"} } }'
     elif [[ $@ == "-X POST jobs/30/comments text=Starting investigation for job 31" ]]; then
         echo '{"id": 1234}'
     elif [[ $@ == $'-X PUT jobs/30/comments/1234 text=Automatic investigation jobs for job 31:\n\nfoo' ]]; then
@@ -47,10 +53,15 @@ openqa-cli() {
         echo '{"id": 1237}'
     elif [[ $@ == "-X GET jobs/32/comments" ]]; then
         echo '[{"id": 1236, "text":"Starting investigation for job 32"},{"id": 1237, "text":"Starting investigation for job 32"}]'
+    elif [[ $@ =~ "-X POST jobs/35/comments" ]]; then
+        warn "Commenting 35 ($@)"
+        exit 99
     elif [[ $@ == '--apibase  --json tests/27/dependencies_ajax' ]]; then
         echo '{"cluster":{}, "edges":[], "nodes":[{"id":27,"state":"done","result":"passed"}]}'
     elif [[ $@ == '--apibase  --json tests/28/dependencies_ajax' ]]; then
         echo '{"cluster":{}, "edges":[], "nodes":[{"id":28,"state":"done","result":"failed"}]}'
+    elif [[ $@ == '--apibase  --json tests/33/dependencies_ajax' ]]; then
+        echo '{"cluster":{}, "edges":[], "nodes":[]}'
     elif [[ $@ == '--apibase  --json tests/29/dependencies_ajax' ]]; then
         echo '{"cluster":{"cluster_foo":[28],"cluster_bar":[29]}, "edges":[], "nodes":[{"id":28,"state":"done","result":"failed"},{"id":29,"state":"done","result":"passed"}]}'
     elif [[ $@ == '--apibase  --json tests/30/dependencies_ajax' ]]; then
@@ -94,6 +105,20 @@ try force=true investigate 31
 is "$rc" 0 'success when job is skipped (because of exclude_no_group and job w/o group)'
 has "$got" 'Job w/o job group, $exclude_no_group is set, skipping investigation'
 
+try investigate 33
+is "$rc" 0 'success (33)'
+has "$got" "Job is ':investigate:' already, skipping investigation" "skip :investigate: (33)"
+
+try investigate 34
+is "$rc" 2 'mocked function returned failure (34)'
+has "$got" "Commenting 35" "Posting comment on OPENQA_INVESTIGATE_ORIGIN (34)"
+has "$got" "likely not a sporadic" "not sporadic (34)"
+
+try investigate 35
+is "$rc" 2 'mocked function returned failure (35)'
+has "$got" "Commenting 35" "Posting comment on OPENQA_INVESTIGATE_ORIGIN (35)"
+has "$got" "likely a sporadic" "sporadic (35)"
+
 # test syncing via investigation comment; we're first
 try force=true sync_via_investigation_comment 31 30
 is "$rc" 255 'do not skip if we own first investigation comment'

test/03-openqa-label-known-issues-and-investigate-hook.t

@@ -2,7 +2,7 @@
 
 source test/init
 
-plan tests 5
+plan tests 18
 dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 
 source "$dir/../openqa-label-known-issues-and-investigate-hook"
@@ -17,20 +17,55 @@ openqa-trigger-bisect-jobs() {
 }
 openqa-label-known-issues() {
     testurl=$1
+    warn "- openqa-label-known-issues $testurl"
     echo "[$testurl]($testurl): Unknown issue, to be reviewed -> $testurl/file/autoinst-log.txt"
 }
 openqa-investigate() {
     local testurl=$1
     "$INVESTIGATE_FAIL" && return 1
     "$INVESTIGATE_RETRIGGER_HOOK" && return 142
-    echo "openqa-investigate $testurl"
+    warn "- openqa-investigate $testurl"
+}
+openqa-trigger-bisect-jobs() {
+    warn "- openqa-trigger-bisect-jobs $1"
+}
+openqa-api-get() {
+    local path=$1
+    if [[ "$path" == "jobs/123" ]]; then
+        echo '{"job":{"state":"done", "result":"failed", "test":"foo"}}'
+    elif [[ "$path" == "jobs/124" ]]; then
+        echo '{"job":{"state":"done", "result":"passed", "test":"foo"}}'
+    elif [[ "$path" == "jobs/125" ]]; then
+        echo '{"job":{"state":"done", "result":"failed", "test":"foo:investigate:retry:x"}}'
+    elif [[ "$path" == "jobs/126" ]]; then
+        echo '{"job":{"state":"done", "result":"failed", "test":"foo:investigate:abc:x"}}'
+    fi
 }
 
 try hook 123
-is "$rc" 0 'successful hook'
+is "$rc" 0 'successful hook (123)'
+has "$got" "- openqa-label-known-issues"
+has "$got" "- openqa-investigate"
+has "$got" "- openqa-trigger-bisect-jobs"
+
+try hook 124
+is "$rc" 0 'successful hook (124)'
+hasnt "$got" "- openqa-label-known-issues"
+has "$got" "- openqa-investigate"
+has "$got" "- openqa-trigger-bisect-jobs"
+
+try hook 125
+is "$rc" 0 'successful hook (125)'
+has "$got" "- openqa-label-known-issues"
+has "$got" "- openqa-investigate"
+has "$got" "- openqa-trigger-bisect-jobs"
+
+try hook 126
+is "$rc" 0 'successful hook (126)'
+has "$got" "- openqa-label-known-issues"
+has "$got" "- openqa-investigate"
+has "$got" "- openqa-trigger-bisect-jobs"
 
-has "$got" 'openqa-investigate https://openqa.opensuse.org/tests/123' 'correct output 1'
-has "$got" 'openqa-trigger-bisect-jobs (--url https://openqa.opensuse.org/tests/123)' 'correct output 2'
 
 export INVESTIGATE_FAIL=true
 try hook 123