[CI] Single button press for automated Qase Test Runs

.gitignore

@@ -24,11 +24,11 @@ scripts/utils/export-metrics/*.tar.gz
 
 # Binaries
 /dartboard
-/scripts/soak/soak
-/qasereporter-k6/qasereporter-k6
+qase-k6-cli/qase-k6-cli
 
 # Vendored binaries
 /internal/vendored/bin
 /.bin
 
 dist/
+.env

CI/k6-runner.groovy

@@ -130,7 +130,7 @@ pipeline {
             def k6EnvContent = """
 BASE_URL=${baseURL}
 KUBECONFIG=${kubeconfigContainerPath ? kubeconfigContainerPath : ''}
-K6_TEST=${params.K6_TEST_FILE}
+K6_REPORT_PREFIX=${params.K6_TEST_FILE}
 K6_NO_USAGE_REPORT=true
 ${params.K6_ENV}
 """

Dockerfile

@@ -18,11 +18,11 @@ RUN go mod download && \
 RUN cd $WORKSPACE && \
     make && \
     mv ./dartboard /usr/local/bin/dartboard && \
-    mv ./qasereporter-k6/qasereporter-k6 /usr/local/bin/qasereporter-k6
+    mv ./qase-k6-cli/qase-k6-cli /usr/local/bin/qase-k6-cli
 
 FROM grafana/k6:${K6_VERSION}
 COPY --from=builder /usr/local/bin/dartboard /bin/dartboard
-COPY --from=builder /usr/local/bin/qasereporter-k6 /bin/qasereporter-k6
+COPY --from=builder /usr/local/bin/qase-k6-cli /bin/qase-k6-cli
 
 # Run the following commands as root user so that we can easily install some needed tools
 USER root

Makefile

@@ -1,5 +1,5 @@
 DARTBOARD_BIN_NAME := dartboard
-REPORTER_BIN_NAME := qasereporter-k6
+QASE_K6_CLI_BIN_NAME := qase-k6-cli
 LDFLAGS := -w -s
 TOFU := ./internal/vendored/bin/tofu
 TOFU_MAIN_DIRS := k3d aws azure harvester
@@ -9,21 +9,21 @@ TOFU_MAIN_DIRS := k3d aws azure harvester
 # =============================================================================
 
 .PHONY: build
-build: internal/vendored/bin qasereporter-k6/${REPORTER_BIN_NAME}
+build: internal/vendored/bin qase-k6-cli/${QASE_K6_CLI_BIN_NAME}
 	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o ${DARTBOARD_BIN_NAME} cmd/dartboard/*.go
 
 internal/vendored/bin:
 	sh download-vendored-bin.sh
 
-qasereporter-k6/${REPORTER_BIN_NAME}: qasereporter-k6/*.go
-	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $@ qasereporter-k6/*.go
+qase-k6-cli/${QASE_K6_CLI_BIN_NAME}: qase-k6-cli/*.go
+	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $@ qase-k6-cli/*.go
 
 .PHONY: clean
 clean:
 	rm -rfv .bin
 	rm -rfv internal/vendored/bin
 	rm -fv ${DARTBOARD_BIN_NAME}
-	rm -fv qasereporter-k6/${REPORTER_BIN_NAME}
+	rm -fv qase-k6-cli/${QASE_K6_CLI_BIN_NAME}
 
 # =============================================================================
 # Go module verification

README.md

@@ -106,11 +106,11 @@ You can find this summary in the form of a `.html` file on the build's page in J
 
 See the [docs](docs) directory for a list of tests that were run with previous versions of this code and their results.
 
-## Qase k6 Reporter
+## Qase k6 CLI
 
-The `qasereporter-k6` utility is a command-line tool included as a separate golang module that parses the output of a k6 test run and reports the results to a test case wtihin a Qase test run. It can be used in CI/CD pipelines to automatically update test cases in Qase with the results from k6 performance tests.
+The `qase-k6-cli` utility is a command-line tool included as a separate golang module that bridges k6 and Qase. It can gather test case information and report k6 test results to Qase.
 
-For detailed usage instructions, including environment variables and command-line flags, please see the [qasereporter-k6 README](qasereporter-k6/README.md).
+For detailed usage instructions, including environment variables and command-line flags, please see the [qase-k6-cli README](qase-k6-cli/README.md).
 
 ## Common Troubleshooting
 

internal/qase/client.go

@@ -19,6 +19,7 @@ import (
 const (
 	TestRunNameEnvVar  = "QASE_TEST_RUN_NAME"
 	TestCaseNameEnvVar = "QASE_TEST_CASE_NAME"
+	TestCaseIDEnvVar   = "QASE_TEST_CASE_ID"
 )
 
 // CustomUnifiedClient combines V1 and V2 clients for our specific needs.
@@ -112,14 +113,18 @@ func (c *CustomUnifiedClient) CreateTestRun(ctx context.Context, testRunName, pr
 }
 
 // GetTestRun retrieves a Qase test run by its ID.
-func (c *CustomUnifiedClient) GetTestRun(ctx context.Context, projectCode string, runID int64) (*api_v1_client.Run, error) {
+func (c *CustomUnifiedClient) GetTestRun(ctx context.Context, projectCode string, runID int64, include *string) (*api_v1_client.Run, error) {
 	logrus.Debugf("Getting test run with ID %d in project %s", runID, projectCode)
 
 	authCtx := context.WithValue(ctx, api_v1_client.ContextAPIKeys, map[string]api_v1_client.APIKey{
 		"TokenAuth": {Key: c.Config.TestOps.API.Token},
 	})
 
-	resp, res, err := c.V1Client.GetAPIClient().RunsAPI.GetRun(authCtx, projectCode, int32(runID)).Execute()
+	apiRunRequest := c.V1Client.GetAPIClient().RunsAPI.GetRun(authCtx, projectCode, int32(runID))
+	if include != nil {
+		apiRunRequest = apiRunRequest.Include(*include)
+	}
+	resp, res, err := apiRunRequest.Execute()
 	logResponseBody(res, "GetTestRun")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get test run: %w", err)
@@ -232,6 +237,23 @@ func (c *CustomUnifiedClient) GetTestCaseByTitle(ctx context.Context, projectCod
 	return matchingCase, nil
 }
 
+// GetCustomFields retrieves all custom fields.
+func (c *CustomUnifiedClient) GetCustomFields(ctx context.Context) (*api_v1_client.CustomFieldListResponse, error) {
+	logrus.Debug("Getting custom fields")
+
+	authCtx := context.WithValue(ctx, api_v1_client.ContextAPIKeys, map[string]api_v1_client.APIKey{
+		"TokenAuth": {Key: c.Config.TestOps.API.Token},
+	})
+
+	resp, res, err := c.V1Client.GetAPIClient().CustomFieldsAPI.GetCustomFields(authCtx).Execute()
+	logResponseBody(res, "GetCustomFields")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get custom fields: %w", err)
+	}
+
+	return resp, nil
+}
+
 // CreateTestResultV1 creates a test result using the V1 API.
 func (c *CustomUnifiedClient) CreateTestResultV1(ctx context.Context, projectCode string, runID int64, result api_v1_client.ResultCreate) error {
 	authCtx := context.WithValue(ctx, api_v1_client.ContextAPIKeys, map[string]api_v1_client.APIKey{

internal/qase/status.go

@@ -1,10 +1,11 @@
 package qase
 
 const (
-	StatusPassed  = "passed"
-	StatusFailed  = "failed"
-	StatusBlocked = "blocked"
-	StatusSkipped = "skipped"
-	StatusInvalid = "invalid"
-	StatusError   = "error"
+	StatusPassed             = "passed"
+	StatusFailed             = "failed"
+	StatusBlocked            = "blocked"
+	StatusSkipped            = "skipped"
+	StatusInvalid            = "invalid"
+	StatusError              = "error"
+	StatusExceededThresholds = "exceeded-thresholds"
 )

k6/crds/create_crds.js

@@ -22,6 +22,7 @@ export const timePolled = new Trend('time_polled', true);
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   scenarios: {
     create: {
       executor: 'shared-iterations',

k6/crds/delete_crds.js

@@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true);
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   scenarios: {
     delete: {
       executor: 'shared-iterations',

k6/crds/get_crds.js

@@ -20,6 +20,7 @@ export const handleSummary = k6Util.customHandleSummary;
 
 // Option setting
 export const options = {
+  insecureSkipTLSVerify: true,
   setupTimeout: '8h',
   scenarios: {
     get: {

k6/crds/load_crds.js

@@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true);
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   scenarios: {
     load: {
       executor: 'shared-iterations',

k6/crds/update_crds.js

@@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true);
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   scenarios: {
     update: {
       executor: 'shared-iterations',

k6/crds/update_destructive_crds.js

@@ -23,6 +23,7 @@ export const timePolled = new Trend('time_polled', true);
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   scenarios: {
     destructiveUpdate: {
       executor: 'shared-iterations',

k6/generic/k6_utils.js

@@ -138,7 +138,7 @@ export function createReports(prefix, data) {
  * A pre-configured `handleSummary` function that generates multiple reports.
  *
  * This function is a convenient wrapper around `createReports`. It automatically
- * determines the report filename prefix from the `K6_TEST` environment variable.
+ * determines the report filename prefix from the `K6_REPORT_PREFIX` environment variable.
  *
  * To use this, import it then export it as `handleSummary`:
  * ```javascript
@@ -158,6 +158,6 @@ export function createReports(prefix, data) {
  * @returns {object} An object mapping filenames to report content, for k6 to write to disk.
  */
 export function customHandleSummary(data) {
-  const prefix = __ENV.K6_TEST ? __ENV.K6_TEST.replace(/\.js$/, '') + "-" : '';
+  const prefix = __ENV.K6_REPORT_PREFIX ? __ENV.K6_REPORT_PREFIX.replace(/\.js$/, '') + "-" : '';
   return createReports(getPathBasename(prefix), data)
 }

k6/generic/report_generator.js

@@ -0,0 +1,23 @@
+import { customHandleSummary } from './k6_utils.js';
+
+const summaryPath = __ENV.K6_SUMMARY_JSON_FILE;
+
+// Load summary data during initialization
+let data = null;
+if (summaryPath) {
+    try {
+        data = JSON.parse(open(summaryPath));
+    } catch (e) {
+        console.error(`Failed to parse summary JSON from ${summaryPath}: ${e.message}`);
+    }
+}
+
+export function handleSummary() {
+    if (!data) {
+        console.log("No summary data available to generate reports.");
+        return {};
+    }
+    return customHandleSummary(data);
+}
+
+export default function() {}

k6/schemas/verify_schemas.js

@@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true);
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'count'],
   scenarios: {
     verify: {

k6/tests/load_projects_rbac.js

@@ -39,6 +39,7 @@ let createdNamespaceIds = []
 export const handleSummary = k6Util.customHandleSummary;
 
 export const options = {
+  insecureSkipTLSVerify: true,
   scenarios: {
     load: {
       executor: 'per-vu-iterations',
@@ -128,7 +129,6 @@ export const options = {
   },
   setupTimeout: '30m',
   teardownTimeout: '30m',
-  insecureSkipTLSVerify: true,
   // httpDebug: 'full',
 }
 

qasereporter-k6/README.md → qase-k6-cli/README.md

@@ -1,6 +1,6 @@
-# k6 Qase Reporter
+# Qase k6 CLI
 
-`qasereporter-k6` is a command-line tool that parses the output of a k6 test run and reports the results to a Qase test run.
+`qase-k6-cli` is a command-line tool that integrates k6 with Qase. It provides functionality to gather test case information from Qase and report k6 test results back to Qase.
 
 It supports two modes of operation for parsing k6 results:
 
@@ -10,17 +10,21 @@ It supports two modes of operation for parsing k6 results:
 
 ## Build
 
-You can build the reporter using the `make` target from the root of the `dartboard` repository:
+You can build the cli using the `make` target from the root of the `dartboard` repository:
 
 ```shell
-make qasereporter-k6
+make qase-k6-cli
 ```
 
-This will produce a `qasereporter-k6` binary inside the `qasereporter-k6/` directory.
+This will produce a `qase-k6-cli` binary inside the `qase-k6-cli/` directory.
 
 ## Usage
 
-The reporter is configured via environment variables and command-line flags.
+The tool is configured via environment variables and command-line flags and has two subcommands: `report` and `gather`.
+
+### Report
+
+Parses k6 metrics/summary and reports the result to Qase.
 
 ```shell
 # Example for Summary Mode
@@ -32,7 +36,7 @@ export QASE_TEST_CASE_NAME="My test case"
 export K6_SUMMARY_JSON_FILE="/path/to/summary.json"
 export K6_SUMMARY_HTML_FILE="/path/to/report.html" # Optional
 
-./qasereporter-k6
+./qase-k6-cli report
 
 # Example for Granular Mode
 export QASE_TESTOPS_API_TOKEN=TOKEN
@@ -42,7 +46,18 @@ export QASE_TEST_CASE_NAME="My k6 Test"
 export K6_OUTPUT_FILE="/path/to/k6-metrics-output.json"
 export K6_SUMMARY_HTML_FILE="/path/to/report.html" # Optional
 
-./qasereporter-k6 -granular
+./qase-k6-cli report -granular
+```
+
+### Gather
+
+Retrieves test cases from a Qase test run and outputs the 'AutomationTestName' custom field value for each case as JSON.
+
+```shell
+export QASE_TESTOPS_API_TOKEN=TOKEN
+export QASE_TESTOPS_PROJECT=PRJ
+
+./qase-k6-cli gather -runID 42
 ```
 
 ### Environment Variables
@@ -57,7 +72,7 @@ export K6_SUMMARY_HTML_FILE="/path/to/report.html" # Optional
 | `K6_SUMMARY_JSON_FILE`   | Path to the k6 summary JSON file. (Used in **Summary Mode**).                                             | For Summary Mode                 |
 | `K6_SUMMARY_HTML_FILE`   | (Optional) Path to the k6 HTML report file to be attached to the Qase result. (Used in **Summary Mode**). | No                               |
 | `K6_OUTPUT_FILE`         | Path to the k6 raw JSON output file. (Used in **Granular Mode**).                                         | For Granular Mode                |
-| `QASE_DEBUG`             | A string ("true" or "false") that enables or disables debug logs.                                          | No                               |
+| `QASE_DEBUG`             | A string ("true" or "false") that enables or disables debug logs.                                         | No                               |
 
 ### Command-line Flags
 

qasereporter-k6/build_qase_reporter_k6.sh → qase-k6-cli/build_qase_k6_cli.sh

@@ -1,18 +1,18 @@
 #!/usr/bin/env bash
 set -e
 
-# Path to the Go source code for the k6 reporter
-REPORTER_SRC_DIR="$(dirname "$0")"
+# Path to the Go source code for the qase-k6-cli
+QASE_K6_CLI_SRC_DIR="$(dirname "$0")"
 
 # Output binary path
-OUTPUT_BINARY="${REPORTER_SRC_DIR}/reporter-k6"
+OUTPUT_BINARY="${QASE_K6_CLI_SRC_DIR}/qase-k6-cli"
 
-echo "Building k6 Qase reporter..."
-echo "Source directory: ${REPORTER_SRC_DIR}"
+echo "Building qase-k6-cli..."
+echo "Source directory: ${QASE_K6_CLI_SRC_DIR}"
 echo "Output binary: ${OUTPUT_BINARY}"
 
 # Ensure we are in the correct directory to resolve modules
-cd "${REPORTER_SRC_DIR}"
+cd "${QASE_K6_CLI_SRC_DIR}"
 
 # Tidy and build the Go application
 go mod tidy