diff --git a/.gitignore b/.gitignore index 2dde72219..cbc6b0dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,11 +24,11 @@ scripts/utils/export-metrics/*.tar.gz # Binaries /dartboard -/scripts/soak/soak -/qasereporter-k6/qasereporter-k6 +qase-k6-cli/qase-k6-cli # Vendored binaries /internal/vendored/bin /.bin dist/ +.env diff --git a/CI/k6-runner.groovy b/CI/k6-runner.groovy index 81227617d..d3d717749 100644 --- a/CI/k6-runner.groovy +++ b/CI/k6-runner.groovy @@ -130,7 +130,7 @@ pipeline { def k6EnvContent = """ BASE_URL=${baseURL} KUBECONFIG=${kubeconfigContainerPath ? kubeconfigContainerPath : ''} -K6_TEST=${params.K6_TEST_FILE} +K6_REPORT_PREFIX=${params.K6_TEST_FILE} K6_NO_USAGE_REPORT=true ${params.K6_ENV} """ diff --git a/CI/qase-k6-runner.groovy b/CI/qase-k6-runner.groovy new file mode 100644 index 000000000..6c808746b --- /dev/null +++ b/CI/qase-k6-runner.groovy @@ -0,0 +1,377 @@ +#!groovy +// Declarative Pipeline Syntax +@Library('qa-jenkins-library') _ + +def agentLabel = 'jenkins-qa-jenkins-agent' +if (params.JENKINS_AGENT_LABEL) { + agentLabel = params.JENKINS_AGENT_LABEL +} + +def kubeconfigContainerPath +def baseURL +def sanitizeCharacterRegex = "[^a-zA-Z0-9'_-]" +def sanitizeK6EnvRegex = "[^a-zA-Z0-9_=,;&*-.\\n\\r]" + +pipeline { + agent { label agentLabel } + + environment { + IMAGE_NAME = 'dartboard' + ARTIFACTS_DIR = 'deployment-artifacts' + ACCESS_LOG = 'access-details.log' + KUBECONFIG_FILE = 'upstream.yaml' + // QASE_TESTOPS_PROJECT and QASE_TESTOPS_RUN_ID are expected as build parameters + } + + stages { + stage('Checkout') { + steps { + script { + project.checkout(repository: params.REPO, branch: params.BRANCH, target: 'dartboard') + } + } + } + + stage('Set Build Description') { + steps { + script { + def testRun = params.QASE_TESTOPS_PROJECT && params.QASE_TESTOPS_RUN_ID ? "${params.QASE_TESTOPS_PROJECT}-${params.QASE_TESTOPS_RUN_ID}": '' + currentBuild.description = "${testRun}" + } + } + } + + stage('Prepare Environment from S3') { + when { expression { return params.DEPLOYMENT_ID } } + steps { + dir('dartboard') { + script { + property.useWithCredentials(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + // Sanitize inputs to prevent shell injection + def safeRegion = (params.S3_BUCKET_REGION ?: "").replaceAll(sanitizeCharacterRegex, "") + def safeBucket = (params.S3_BUCKET_NAME ?: "").replaceAll(sanitizeCharacterRegex, "") + def safeDeploymentId = (params.DEPLOYMENT_ID ?: "").replaceAll(sanitizeCharacterRegex, "") + + withEnv(["SAFE_REGION=${safeRegion}", "SAFE_BUCKET=${safeBucket}", "SAFE_DEPLOYMENT_ID=${safeDeploymentId}"]) { + sh """ + mkdir -p ${env.ARTIFACTS_DIR} + docker run --rm \\ + -v "${pwd()}/${env.ARTIFACTS_DIR}:/artifacts" \\ + -e AWS_ACCESS_KEY_ID \\ + -e AWS_SECRET_ACCESS_KEY \\ + -e AWS_S3_REGION="\${SAFE_REGION}" \\ + amazon/aws-cli s3 cp "s3://\${SAFE_BUCKET}/\${SAFE_DEPLOYMENT_ID}/" /artifacts/ --recursive + + # Unzip the config archive + config_zip=\$(find ${env.ARTIFACTS_DIR} -name '*_config.zip' | head -n 1) + if [ -n "\$config_zip" ]; then + unzip -o "\$config_zip" -d "${env.ARTIFACTS_DIR}" + else + echo "Warning: No config zip file found in S3 artifacts." + fi + + echo "Downloaded artifacts:" + ls -l ${env.ARTIFACTS_DIR} + """ + } + } + // Extract FQDN and set environment variables for the next stage + def accessLogPath = "./${env.ARTIFACTS_DIR}/${env.ACCESS_LOG}" + if (fileExists(accessLogPath)) { + def accessLogContent = readFile(accessLogPath) + // See https://docs.groovy-lang.org/next/html/groovy-jdk/java/util/regex/Matcher.html + def matcher = accessLogContent =~ /(?m)^\s*Rancher UI:\s*(https?:\/\/[^ :]+)/ + if (matcher.find()) { + def match = matcher.group(1).trim() + baseURL = "${match}" + echo "Found Rancher URL: ${baseURL}" + } else { + echo "Warning: Could not find 'Rancher UI' in ${env.ACCESS_LOG}" + } + } + + // Find the upstream.yaml file within the downloaded artifacts and move it to the current directory. + // This is more robust than assuming its exact location after unzipping. + sh """ + kubeconfig_file=\$(find ./${env.ARTIFACTS_DIR} -name '${env.KUBECONFIG_FILE}' -print -quit) + if [ -n "\$kubeconfig_file" ]; then + mv "\$kubeconfig_file" "./${env.KUBECONFIG_FILE}" + fi + """ + def kubeconfigPath = "./${env.KUBECONFIG_FILE}" + if (fileExists(kubeconfigPath)) { + // Absolute path relative to the container's filespace + kubeconfigContainerPath = "/app/${env.KUBECONFIG_FILE}" + echo "Found kubeconfig at: ${kubeconfigPath}" + } + } + } + } + } + + stage('Build Dartboard Image') { + steps { + dir('dartboard') { + sh "docker build -t ${env.IMAGE_NAME}:latest ." + } + } + } + + stage('Gather Test Cases') { + steps { + dir('dartboard') { + script { + // Sanitize Run ID to be numeric only to prevent command injection + def safeRunID = (params.QASE_TESTOPS_RUN_ID ?: "").replaceAll("[^0-9]", "") + def safeProject = (params.QASE_TESTOPS_PROJECT ?: "").replaceAll(sanitizeCharacterRegex, "") + + withEnv(["QASE_PROJECT=${safeProject}", "QASE_RUN_ID=${safeRunID}"]) { + withCredentials([string(credentialsId: "QASE_AUTOMATION_TOKEN", variable: "QASE_TESTOPS_API_TOKEN")]) { + sh """ + docker run --rm --name dartboard-qase-gatherer \\ + -v "${pwd()}:/app" \\ + --user=\$(id -u) \\ + --workdir /app \\ + --entrypoint='' \\ + -e QASE_TESTOPS_API_TOKEN \\ + -e QASE_TESTOPS_PROJECT="\${QASE_PROJECT}" \\ + ${env.IMAGE_NAME}:latest qase-k6-cli gather -runID "\${QASE_RUN_ID}" > test_cases.json + """ + } + } + sh "cat test_cases.json" + } + } + } + } + + stage('Run Tests') { + steps { + dir('dartboard') { + script { + // Get the number of test cases using yq + def countStr = sh(script: """ + docker run --rm \\ + -v "${pwd()}:/app" \\ + --workdir /app \\ + --user=\$(id -u) \\ + --entrypoint='' \\ + ${env.IMAGE_NAME}:latest yq '. | length' test_cases.json + """, returnStdout: true).trim() + + def count = countStr.isInteger() ? countStr.toInteger() : 0 + + if (count == 0) { + echo "No test cases found with 'AutomationTestName' custom field in Run ID ${params.QASE_TESTOPS_RUN_ID}." + return + } + + // Iterate over each gathered test case + for (int index = 0; index < count; index++) { + // Extract test case details using yq + // We output ID, Title, ScriptPath, and then parameters as key=::=value lines + def caseDataLines = sh(script: """ + docker run --rm \\ + -v "${pwd()}:/app" \\ + --workdir /app \\ + --user=\$(id -u) \\ + --entrypoint='' \\ + ${env.IMAGE_NAME}:latest yq -r ".[${index}] | (.id, .title, .automation_test_name, (.parameters // {} | to_entries | .[] | \\"\\(.key)=::=\\(.value)\\"))" test_cases.json + """, returnStdout: true).trim().readLines() + + if (caseDataLines.size() < 3) { + echo "Warning: Could not parse test case at index ${index}" + continue + } + + def caseId = caseDataLines[0] + def caseTitle = caseDataLines[1] + def scriptPath = caseDataLines[2] + def parameters = [:] + + if (caseDataLines.size() > 3) { + caseDataLines[3..-1].each { line -> + def parts = line.split('=::=', 2) + if (parts.length == 2) { + parameters[parts[0]] = parts[1] + } + } + } + + echo "-------------------------------------------------------" + echo "Processing Case ID: ${caseId}" + echo "Title: ${caseTitle}" + echo "Script: ${scriptPath}" + echo "Parameters: ${parameters}" + echo "-------------------------------------------------------" + + // Sanitize project name to prevent shell injection in filenames + def safeProject = (params.QASE_TESTOPS_PROJECT ?: "").replaceAll(sanitizeCharacterRegex, "") + + // 1. Prepare Environment for this specific test case + // Use index to ensure uniqueness for file names when multiple parameter combinations exist for the same case ID + def basename = sh(script: "basename ${scriptPath}", returnStdout: true).trim().replaceAll("\\.js", "") + def envFile = "k6-${basename}-${safeProject}-${caseId}-${index}.env" + def k6ReportPrefix = "k6-${basename}-${safeProject}-${caseId}-${index}" + def summaryLog = "k6-summary-${safeProject}-${caseId}-${index}-params.log" + def summaryJson = "${k6ReportPrefix}-summary.json" + def htmlReport = "${k6ReportPrefix}-summary.html" + def webDashboardReport = "k6-web-dashboard-${safeProject}-${caseId}-${index}.html" + def safeK6Env = params.K6_ENV ? params.K6_ENV.replaceAll(sanitizeK6EnvRegex, "") : "" + + // Construct environment variables content + // We set QASE_TEST_CASE_ID for the reporter + def envContent = "" + + // Handle parameters required by the test case + parameters.each { paramName, paramValue -> + // Check if the Jenkins job has this parameter defined, otherwise use the value from Qase + def finalValue = params[paramName] ?: paramValue + // Sanitize value to prevent newlines breaking the env file format + finalValue = finalValue.toString().replaceAll("[\r\n]", "") + envContent += "${paramName}=${finalValue}\n" + } + + envContent += """ +K6_NO_USAGE_REPORT=true +K6_TEST=${scriptPath} +K6_REPORT_PREFIX=${k6ReportPrefix} +BASE_URL=${baseURL ?: ''} +KUBECONFIG=${kubeconfigContainerPath ?: ''} +QASE_TESTOPS_PROJECT=${params.QASE_TESTOPS_PROJECT} +QASE_TESTOPS_RUN_ID=${params.QASE_TESTOPS_RUN_ID} +QASE_TEST_CASE_ID=${caseId} +K6_SUMMARY_JSON_FILE=${summaryJson} +K6_SUMMARY_HTML_FILE=${htmlReport} +K6_WEB_DASHBOARD=true +K6_WEB_DASHBOARD_EXPORT=${webDashboardReport} +${safeK6Env} +""" + + writeFile file: envFile, text: envContent + + echo "Environment file for Case ${caseId} (index ${index}):" + echo envContent + + // 2. Run k6 + try { + sh """ + docker run --rm --name dartboard-k6-runner-${index} \\ + -v "${pwd()}:/app" \\ + --env-file "${envFile}" \\ + --workdir /app \\ + --user=\$(id -u) \\ + --entrypoint='' \\ + ${env.IMAGE_NAME}:latest sh -c ''' + set -o pipefail + + # 1. Prepare a script copy with handleSummary disabled to allow Native Dashboard generation + TEST_DIR=\$(dirname "\$K6_TEST") + TEST_FILE=\$(basename "\$K6_TEST") + MODIFIED_TEST="\${TEST_DIR}/native_\${TEST_FILE}" + + cp "\$K6_TEST" "\$MODIFIED_TEST" + # Comment out the export of handleSummary + sed -i "s|export .*handleSummary|// &|" "\$MODIFIED_TEST" + + echo "Running k6 script (Native Dashboard Mode): \$MODIFIED_TEST" + k6 run --no-color "\$MODIFIED_TEST" | tee "${summaryLog}" + + # 2. Generate Custom Reports (k6-reporter, custom JUnit) from the JSON summary + echo "Generating custom reports from ${summaryJson}..." + # Use absolute path for summary file as open() in the script resolves relative to the script location + k6 run --no-color -e K6_SUMMARY_JSON_FILE="/app/${summaryJson}" k6/generic/report_generator.js > /dev/null + ''' + """ + } catch (Exception e) { + echo "k6 run failed for case ${caseId}, but continuing to report failure/partial results." + } + sh "ls -al" + + // 3. Report to Qase + withCredentials([string(credentialsId: "QASE_AUTOMATION_TOKEN", variable: "QASE_TESTOPS_API_TOKEN")]) { + sh """ + docker run --rm --name dartboard-qase-reporter-${index} \\ + -v "${pwd()}:/app" \\ + --env-file "${envFile}" \\ + --workdir /app \\ + --user=\$(id -u) \\ + --entrypoint='' \\ + -e QASE_TESTOPS_API_TOKEN \\ + ${env.IMAGE_NAME}:latest sh -c ''' + echo "Reporting results for Case ${caseId}..." + if [ -f "${summaryJson}" ]; then + qase-k6-cli report + else + echo "Summary JSON not found, skipping report for ${caseId}" + fi + ''' + """ + } + } + } + } + } + } + } + + post { + always { + script { + echo "Archiving k6 test results..." + archiveArtifacts artifacts: """ + dartboard/*.json, + dartboard/*.log, + dartboard/*.html, + dartboard/*.xml, + """.trim(), fingerprint: true + + // The k6 container is run with --rm, so it should clean itself up. + // But if the job is aborted, the container might be left running. + echo "Cleaning up Docker resources..." + try { + echo "Attempting to remove containers matching: dartboard-k6-runner" + sh "docker ps -a -q --filter name=dartboard-k6-runner | xargs -r docker rm -f" + } catch (e) { + echo "Could not remove containers matching 'dartboard-k6-runner'. Details: ${e.message}" + } + try { + echo "Attempting to remove container: dartboard-qase-gatherer" + sh "docker rm -f dartboard-qase-gatherer" + } catch (e) { + echo "Could not remove container 'dartboard-qase-gatherer'. It may have already been removed. Details: ${e.message}" + } + try { + echo "Attempting to remove containers matching: dartboard-qase-reporter" + sh "docker ps -a -q --filter name=dartboard-qase-reporter | xargs -r docker rm -f" + } catch (e) { + echo "Could not remove containers matching 'dartboard-qase-reporter'. Details: ${e.message}" + } + try { + echo "Attempting to remove image: ${env.IMAGE_NAME}:latest" + sh "docker rmi -f ${env.IMAGE_NAME}:latest" + echo "Attempting to remove image: amazon/aws-cli" + sh "docker rmi amazon/aws-cli" + } catch (e) { + echo "Could not remove a Docker image. It may have already been removed or was never present. Details: ${e.message}" + } + } + } + cleanup { + // Clean up large files from the workspace to save disk space on the agent. + // These are not part of the archived artifacts but remain in the workspace. + echo "Cleaning up workspace..." + dir('dartboard') { + // Use find and xargs for more robust and efficient cleanup of non-artifact files and directories. + // This removes all files and directories from the checkout except for the archived k6 results. + sh """ + set -x + echo "Removing all non-artifact files and directories..." + find . -mindepth 1 -maxdepth 1 \\ + -not -name '*.html' -not -name '*.json' -not -name '*.log' -not -name '*.xml' \\ + -exec rm -rf {} + + """ + } + } + } +} diff --git a/Dockerfile b/Dockerfile index 583584629..62454acb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,11 +18,11 @@ RUN go mod download && \ RUN cd $WORKSPACE && \ make && \ mv ./dartboard /usr/local/bin/dartboard && \ - mv ./qasereporter-k6/qasereporter-k6 /usr/local/bin/qasereporter-k6 + mv ./qase-k6-cli/qase-k6-cli /usr/local/bin/qase-k6-cli FROM grafana/k6:${K6_VERSION} COPY --from=builder /usr/local/bin/dartboard /bin/dartboard -COPY --from=builder /usr/local/bin/qasereporter-k6 /bin/qasereporter-k6 +COPY --from=builder /usr/local/bin/qase-k6-cli /bin/qase-k6-cli # Run the following commands as root user so that we can easily install some needed tools USER root diff --git a/Makefile b/Makefile index e7b21dac1..992b4e411 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ DARTBOARD_BIN_NAME := dartboard -REPORTER_BIN_NAME := qasereporter-k6 +QASE_K6_CLI_BIN_NAME := qase-k6-cli LDFLAGS := -w -s TOFU := ./internal/vendored/bin/tofu TOFU_MAIN_DIRS := k3d aws azure harvester @@ -9,21 +9,21 @@ TOFU_MAIN_DIRS := k3d aws azure harvester # ============================================================================= .PHONY: build -build: internal/vendored/bin qasereporter-k6/${REPORTER_BIN_NAME} +build: internal/vendored/bin qase-k6-cli/${QASE_K6_CLI_BIN_NAME} CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o ${DARTBOARD_BIN_NAME} cmd/dartboard/*.go internal/vendored/bin: sh download-vendored-bin.sh -qasereporter-k6/${REPORTER_BIN_NAME}: qasereporter-k6/*.go - CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $@ qasereporter-k6/*.go +qase-k6-cli/${QASE_K6_CLI_BIN_NAME}: qase-k6-cli/*.go + CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $@ qase-k6-cli/*.go .PHONY: clean clean: rm -rfv .bin rm -rfv internal/vendored/bin rm -fv ${DARTBOARD_BIN_NAME} - rm -fv qasereporter-k6/${REPORTER_BIN_NAME} + rm -fv qase-k6-cli/${QASE_K6_CLI_BIN_NAME} # ============================================================================= # Go module verification diff --git a/README.md b/README.md index eef1b075a..5ae372590 100644 --- a/README.md +++ b/README.md @@ -106,11 +106,11 @@ You can find this summary in the form of a `.html` file on the build's page in J See the [docs](docs) directory for a list of tests that were run with previous versions of this code and their results. -## Qase k6 Reporter +## Qase k6 CLI -The `qasereporter-k6` utility is a command-line tool included as a separate golang module that parses the output of a k6 test run and reports the results to a test case wtihin a Qase test run. It can be used in CI/CD pipelines to automatically update test cases in Qase with the results from k6 performance tests. +The `qase-k6-cli` utility is a command-line tool included as a separate golang module that bridges k6 and Qase. It can gather test case information and report k6 test results to Qase. -For detailed usage instructions, including environment variables and command-line flags, please see the [qasereporter-k6 README](qasereporter-k6/README.md). +For detailed usage instructions, including environment variables and command-line flags, please see the [qase-k6-cli README](qase-k6-cli/README.md). ## Common Troubleshooting diff --git a/internal/qase/client.go b/internal/qase/client.go index bf6ef8c63..33eafb907 100644 --- a/internal/qase/client.go +++ b/internal/qase/client.go @@ -19,6 +19,7 @@ import ( const ( TestRunNameEnvVar = "QASE_TEST_RUN_NAME" TestCaseNameEnvVar = "QASE_TEST_CASE_NAME" + TestCaseIDEnvVar = "QASE_TEST_CASE_ID" ) // CustomUnifiedClient combines V1 and V2 clients for our specific needs. @@ -112,14 +113,18 @@ func (c *CustomUnifiedClient) CreateTestRun(ctx context.Context, testRunName, pr } // GetTestRun retrieves a Qase test run by its ID. -func (c *CustomUnifiedClient) GetTestRun(ctx context.Context, projectCode string, runID int64) (*api_v1_client.Run, error) { +func (c *CustomUnifiedClient) GetTestRun(ctx context.Context, projectCode string, runID int64, include *string) (*api_v1_client.Run, error) { logrus.Debugf("Getting test run with ID %d in project %s", runID, projectCode) authCtx := context.WithValue(ctx, api_v1_client.ContextAPIKeys, map[string]api_v1_client.APIKey{ "TokenAuth": {Key: c.Config.TestOps.API.Token}, }) - resp, res, err := c.V1Client.GetAPIClient().RunsAPI.GetRun(authCtx, projectCode, int32(runID)).Execute() + apiRunRequest := c.V1Client.GetAPIClient().RunsAPI.GetRun(authCtx, projectCode, int32(runID)) + if include != nil { + apiRunRequest = apiRunRequest.Include(*include) + } + resp, res, err := apiRunRequest.Execute() logResponseBody(res, "GetTestRun") if err != nil { return nil, fmt.Errorf("failed to get test run: %w", err) @@ -232,6 +237,23 @@ func (c *CustomUnifiedClient) GetTestCaseByTitle(ctx context.Context, projectCod return matchingCase, nil } +// GetCustomFields retrieves all custom fields. +func (c *CustomUnifiedClient) GetCustomFields(ctx context.Context) (*api_v1_client.CustomFieldListResponse, error) { + logrus.Debug("Getting custom fields") + + authCtx := context.WithValue(ctx, api_v1_client.ContextAPIKeys, map[string]api_v1_client.APIKey{ + "TokenAuth": {Key: c.Config.TestOps.API.Token}, + }) + + resp, res, err := c.V1Client.GetAPIClient().CustomFieldsAPI.GetCustomFields(authCtx).Execute() + logResponseBody(res, "GetCustomFields") + if err != nil { + return nil, fmt.Errorf("failed to get custom fields: %w", err) + } + + return resp, nil +} + // CreateTestResultV1 creates a test result using the V1 API. func (c *CustomUnifiedClient) CreateTestResultV1(ctx context.Context, projectCode string, runID int64, result api_v1_client.ResultCreate) error { authCtx := context.WithValue(ctx, api_v1_client.ContextAPIKeys, map[string]api_v1_client.APIKey{ diff --git a/internal/qase/status.go b/internal/qase/status.go index a89b44f1b..56394eefe 100644 --- a/internal/qase/status.go +++ b/internal/qase/status.go @@ -1,10 +1,11 @@ package qase const ( - StatusPassed = "passed" - StatusFailed = "failed" - StatusBlocked = "blocked" - StatusSkipped = "skipped" - StatusInvalid = "invalid" - StatusError = "error" + StatusPassed = "passed" + StatusFailed = "failed" + StatusBlocked = "blocked" + StatusSkipped = "skipped" + StatusInvalid = "invalid" + StatusError = "error" + StatusExceededThresholds = "exceeded-thresholds" ) diff --git a/k6/crds/create_crds.js b/k6/crds/create_crds.js index bf893b2da..174bc4016 100644 --- a/k6/crds/create_crds.js +++ b/k6/crds/create_crds.js @@ -22,6 +22,7 @@ export const timePolled = new Trend('time_polled', true); export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, scenarios: { create: { executor: 'shared-iterations', diff --git a/k6/crds/delete_crds.js b/k6/crds/delete_crds.js index 08de28a48..c1e71bb2c 100644 --- a/k6/crds/delete_crds.js +++ b/k6/crds/delete_crds.js @@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true); export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, scenarios: { delete: { executor: 'shared-iterations', diff --git a/k6/crds/load_crds.js b/k6/crds/load_crds.js index 188fb5ace..1087102bd 100644 --- a/k6/crds/load_crds.js +++ b/k6/crds/load_crds.js @@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true); export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, scenarios: { load: { executor: 'shared-iterations', diff --git a/k6/crds/update_crds.js b/k6/crds/update_crds.js index 607ca94eb..fdf645325 100644 --- a/k6/crds/update_crds.js +++ b/k6/crds/update_crds.js @@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true); export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, scenarios: { update: { executor: 'shared-iterations', diff --git a/k6/crds/update_destructive_crds.js b/k6/crds/update_destructive_crds.js index 34beebb7f..8251faea6 100644 --- a/k6/crds/update_destructive_crds.js +++ b/k6/crds/update_destructive_crds.js @@ -23,6 +23,7 @@ export const timePolled = new Trend('time_polled', true); export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, scenarios: { destructiveUpdate: { executor: 'shared-iterations', diff --git a/k6/generic/k6_utils.js b/k6/generic/k6_utils.js index 31f993fd5..9ae0539a3 100644 --- a/k6/generic/k6_utils.js +++ b/k6/generic/k6_utils.js @@ -138,7 +138,7 @@ export function createReports(prefix, data) { * A pre-configured `handleSummary` function that generates multiple reports. * * This function is a convenient wrapper around `createReports`. It automatically - * determines the report filename prefix from the `K6_TEST` environment variable. + * determines the report filename prefix from the `K6_REPORT_PREFIX` environment variable. * * To use this, import it then export it as `handleSummary`: * ```javascript @@ -158,6 +158,6 @@ export function createReports(prefix, data) { * @returns {object} An object mapping filenames to report content, for k6 to write to disk. */ export function customHandleSummary(data) { - const prefix = __ENV.K6_TEST ? __ENV.K6_TEST.replace(/\.js$/, '') + "-" : ''; + const prefix = __ENV.K6_REPORT_PREFIX ? __ENV.K6_REPORT_PREFIX.replace(/\.js$/, '') + "-" : ''; return createReports(getPathBasename(prefix), data) } diff --git a/k6/generic/report_generator.js b/k6/generic/report_generator.js new file mode 100644 index 000000000..261deff7d --- /dev/null +++ b/k6/generic/report_generator.js @@ -0,0 +1,23 @@ +import { customHandleSummary } from './k6_utils.js'; + +const summaryPath = __ENV.K6_SUMMARY_JSON_FILE; + +// Load summary data during initialization +let data = null; +if (summaryPath) { + try { + data = JSON.parse(open(summaryPath)); + } catch (e) { + console.error(`Failed to parse summary JSON from ${summaryPath}: ${e.message}`); + } +} + +export function handleSummary() { + if (!data) { + console.log("No summary data available to generate reports."); + return {}; + } + return customHandleSummary(data); +} + +export default function() {} diff --git a/k6/schemas/verify_schemas.js b/k6/schemas/verify_schemas.js index 0a7c6dbed..350085802 100644 --- a/k6/schemas/verify_schemas.js +++ b/k6/schemas/verify_schemas.js @@ -21,6 +21,7 @@ export const timePolled = new Trend('time_polled', true); export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'count'], scenarios: { verify: { diff --git a/k6/tests/load_projects_rbac.js b/k6/tests/load_projects_rbac.js index 0cd9993de..cdf557fad 100644 --- a/k6/tests/load_projects_rbac.js +++ b/k6/tests/load_projects_rbac.js @@ -39,6 +39,7 @@ let createdNamespaceIds = [] export const handleSummary = k6Util.customHandleSummary; export const options = { + insecureSkipTLSVerify: true, scenarios: { load: { executor: 'per-vu-iterations', @@ -128,7 +129,6 @@ export const options = { }, setupTimeout: '30m', teardownTimeout: '30m', - insecureSkipTLSVerify: true, // httpDebug: 'full', } diff --git a/qasereporter-k6/README.md b/qase-k6-cli/README.md similarity index 85% rename from qasereporter-k6/README.md rename to qase-k6-cli/README.md index 26c879f9e..24a4188b8 100644 --- a/qasereporter-k6/README.md +++ b/qase-k6-cli/README.md @@ -1,6 +1,6 @@ -# k6 Qase Reporter +# Qase k6 CLI -`qasereporter-k6` is a command-line tool that parses the output of a k6 test run and reports the results to a Qase test run. +`qase-k6-cli` is a command-line tool that integrates k6 with Qase. It provides functionality to gather test case information from Qase and report k6 test results back to Qase. It supports two modes of operation for parsing k6 results: @@ -10,17 +10,21 @@ It supports two modes of operation for parsing k6 results: ## Build -You can build the reporter using the `make` target from the root of the `dartboard` repository: +You can build the cli using the `make` target from the root of the `dartboard` repository: ```shell -make qasereporter-k6 +make qase-k6-cli ``` -This will produce a `qasereporter-k6` binary inside the `qasereporter-k6/` directory. +This will produce a `qase-k6-cli` binary inside the `qase-k6-cli/` directory. ## Usage -The reporter is configured via environment variables and command-line flags. +The tool is configured via environment variables and command-line flags and has two subcommands: `report` and `gather`. + +### Report + +Parses k6 metrics/summary and reports the result to Qase. ```shell # Example for Summary Mode @@ -32,7 +36,7 @@ export QASE_TEST_CASE_NAME="My test case" export K6_SUMMARY_JSON_FILE="/path/to/summary.json" export K6_SUMMARY_HTML_FILE="/path/to/report.html" # Optional -./qasereporter-k6 +./qase-k6-cli report # Example for Granular Mode export QASE_TESTOPS_API_TOKEN=TOKEN @@ -42,7 +46,18 @@ export QASE_TEST_CASE_NAME="My k6 Test" export K6_OUTPUT_FILE="/path/to/k6-metrics-output.json" export K6_SUMMARY_HTML_FILE="/path/to/report.html" # Optional -./qasereporter-k6 -granular +./qase-k6-cli report -granular +``` + +### Gather + +Retrieves test cases from a Qase test run and outputs the 'AutomationTestName' custom field value for each case as JSON. + +```shell +export QASE_TESTOPS_API_TOKEN=TOKEN +export QASE_TESTOPS_PROJECT=PRJ + +./qase-k6-cli gather -runID 42 ``` ### Environment Variables @@ -57,7 +72,7 @@ export K6_SUMMARY_HTML_FILE="/path/to/report.html" # Optional | `K6_SUMMARY_JSON_FILE` | Path to the k6 summary JSON file. (Used in **Summary Mode**). | For Summary Mode | | `K6_SUMMARY_HTML_FILE` | (Optional) Path to the k6 HTML report file to be attached to the Qase result. (Used in **Summary Mode**). | No | | `K6_OUTPUT_FILE` | Path to the k6 raw JSON output file. (Used in **Granular Mode**). | For Granular Mode | -| `QASE_DEBUG` | A string ("true" or "false") that enables or disables debug logs. | No | +| `QASE_DEBUG` | A string ("true" or "false") that enables or disables debug logs. | No | ### Command-line Flags diff --git a/qasereporter-k6/build_qase_reporter_k6.sh b/qase-k6-cli/build_qase_k6_cli.sh similarity index 51% rename from qasereporter-k6/build_qase_reporter_k6.sh rename to qase-k6-cli/build_qase_k6_cli.sh index 4f49c8963..4456ce3f7 100644 --- a/qasereporter-k6/build_qase_reporter_k6.sh +++ b/qase-k6-cli/build_qase_k6_cli.sh @@ -1,18 +1,18 @@ #!/usr/bin/env bash set -e -# Path to the Go source code for the k6 reporter -REPORTER_SRC_DIR="$(dirname "$0")" +# Path to the Go source code for the qase-k6-cli +QASE_K6_CLI_SRC_DIR="$(dirname "$0")" # Output binary path -OUTPUT_BINARY="${REPORTER_SRC_DIR}/reporter-k6" +OUTPUT_BINARY="${QASE_K6_CLI_SRC_DIR}/qase-k6-cli" -echo "Building k6 Qase reporter..." -echo "Source directory: ${REPORTER_SRC_DIR}" +echo "Building qase-k6-cli..." +echo "Source directory: ${QASE_K6_CLI_SRC_DIR}" echo "Output binary: ${OUTPUT_BINARY}" # Ensure we are in the correct directory to resolve modules -cd "${REPORTER_SRC_DIR}" +cd "${QASE_K6_CLI_SRC_DIR}" # Tidy and build the Go application go mod tidy diff --git a/qasereporter-k6/main.go b/qase-k6-cli/main.go similarity index 63% rename from qasereporter-k6/main.go rename to qase-k6-cli/main.go index ccb904d67..6ca6b3e24 100644 --- a/qasereporter-k6/main.go +++ b/qase-k6-cli/main.go @@ -25,6 +25,7 @@ var ( runIDStr = os.Getenv(qase_config.QaseTestOpsRunIDEnvVar) runName = os.Getenv(qase.TestRunNameEnvVar) testCaseName = os.Getenv(qase.TestCaseNameEnvVar) + testCaseIDEnv = os.Getenv(qase.TestCaseIDEnvVar) k6MetricsOutputFile = os.Getenv(k6MetricsOutputFileEnvVar) ) @@ -95,12 +96,34 @@ type K6Check struct { } func main() { - logrus.Info("Running k6 QASE reporter") + if len(os.Args) < 2 { + fmt.Printf("Usage: %s [options]\n\n", os.Args[0]) + fmt.Println("Subcommands:") + fmt.Println(" report\tParses k6 metrics/summary and reports the result to Qase.") + fmt.Println(" gather\tRetrieves test cases from a Qase test run and outputs the 'AutomationTestName' custom field value for each case.") + fmt.Println() + logrus.Fatal("expected 'report' or 'gather' subcommands") + } - granularReporting := flag.Bool("granular", false, "Enable granular reporting of all Metric and Point lines from k6 JSON output.") - // The -runID flag allows overriding the test case ID. - runIDOverride := flag.String("runID", "", "Qase test run ID to report results against.") - flag.Parse() + switch os.Args[1] { + case "report": + reportCmd := flag.NewFlagSet("report", flag.ExitOnError) + granularReporting := reportCmd.Bool("granular", false, "Enable granular reporting of all Metric and Point lines from k6 JSON output.") + runIDOverride := reportCmd.String("runID", "", "Qase test run ID to report results against.") + reportCmd.Parse(os.Args[2:]) + runReport(*granularReporting, *runIDOverride) + case "gather": + gatherCmd := flag.NewFlagSet("gather", flag.ExitOnError) + runIDGather := gatherCmd.String("runID", "", "Qase test run ID to gather test cases from.") + gatherCmd.Parse(os.Args[2:]) + runGather(*runIDGather) + default: + logrus.Fatalf("Unknown subcommand: %s", os.Args[1]) + } +} + +func runReport(granularReporting bool, runIDOverride string) { + logrus.Info("Running qase-k6-cli reporter") if runIDStr == "" && runName == "" { logrus.Fatalf("Missing required environment variables for reporting: both %s, and %s", @@ -108,8 +131,8 @@ func main() { } // Use the provided case ID flag, otherwise default to the run ID. - if *runIDOverride != "" { - runIDStr = *runIDOverride + if runIDOverride != "" { + runIDStr = runIDOverride } qaseClient = qase.SetupQaseClient() @@ -132,7 +155,7 @@ func main() { } else { runID = parsedRunID - resp, err := qaseClient.GetTestRun(context.Background(), projectID, runID) + resp, err := qaseClient.GetTestRun(context.Background(), projectID, runID, nil) if err != nil { logrus.Fatalf("Failed to get Qase test run while fetching run title (runID: %v): %v", int32(runID), err) } @@ -140,7 +163,16 @@ func main() { runName = *resp.Title } - if testCaseName != "" { + if testCaseIDEnv != "" { + id, err := strconv.ParseInt(testCaseIDEnv, 10, 64) + if err != nil { + logrus.Fatalf("Invalid QASE_TEST_CASE_ID: %v", err) + } + testCaseID = id + logrus.Infof("Using provided Qase test case ID: %d", testCaseID) + } + + if testCaseID == 0 && testCaseName != "" { logrus.Infof("Fetching Qase test case by title: %s", testCaseName) testCase, err := qaseClient.GetTestCaseByTitle(context.Background(), projectID, testCaseName) @@ -149,11 +181,10 @@ func main() { } testCaseID = *testCase.Id - } else { + } else if testCaseID == 0 { // Fallback or error if no test case name is provided - logrus.Fatalf("%s environment variable not set.", qase.TestCaseNameEnvVar) + logrus.Fatalf("Neither %s nor %s environment variables are set.", qase.TestCaseNameEnvVar, qase.TestCaseIDEnvVar) } - // Get the full test case details to check for parameters testCaseDetails, err := qaseClient.GetTestCase(context.Background(), projectID, testCaseID) if err != nil { @@ -162,13 +193,164 @@ func main() { params := getAndValidateTestCaseParameters(testCaseDetails.Parameters) - if !*granularReporting { + if !granularReporting { reportSummary(params) } else { reportMetrics(params) } } +func runGather(runIDOverride string) { + logrus.Info("Running qase-k6-cli gatherer") + + if projectID == "" { + logrus.Fatalf("Missing required environment variable: %s", qase_config.QaseTestOpsProjectEnvVar) + } + + if runIDOverride == "" { + logrus.Fatal("runID is required for gather subcommand") + } + + runIDVal, err := strconv.ParseInt(runIDOverride, 10, 64) + if err != nil { + logrus.Fatalf("Invalid runID: %v", err) + } + + qaseClient = qase.SetupQaseClient() + + include := "cases" + run, err := qaseClient.GetTestRun(context.Background(), projectID, runIDVal, &include) + if err != nil { + logrus.Fatalf("Failed to get test run: %v", err) + } + + cfResp, err := qaseClient.GetCustomFields(context.Background()) + if err != nil { + logrus.Fatalf("Failed to get custom fields: %v", err) + } + + // Get the ID of the "AutomationTestName" custom field + var automationTestNameID int64 + for _, cf := range cfResp.Result.Entities { + if cf.Title != nil && *cf.Title == "AutomationTestName" { + automationTestNameID = *cf.Id + break + } + } + + if automationTestNameID == 0 { + logrus.Fatalf("Custom field 'AutomationTestName' not found for %s", projectID) + } + + type gatheredCase struct { + ID int64 `json:"id"` + Title string `json:"title"` + Parameters map[string]string `json:"parameters"` + AutomationTestName string `json:"automation_test_name"` + } + results := []gatheredCase{} + processedIDs := map[int64]bool{} + + for _, caseID := range run.Cases { + if processedIDs[caseID] { + continue + } + processedIDs[caseID] = true + + tc, err := qaseClient.GetTestCase(context.Background(), projectID, caseID) + if err != nil { + logrus.Errorf("Failed to get test case %d: %v", caseID, err) + continue + } + + paramMap := map[string][]string{} + for _, parameter := range tc.Parameters { + var items []v1.ParameterSingle + if parameter.TestCaseParameterSingle != nil { + items = append(items, parameter.TestCaseParameterSingle.Item) + } else if parameter.TestCaseParameterGroup != nil { + items = append(items, parameter.TestCaseParameterGroup.Items...) + } + for _, item := range items { + val := []string{} + if item.Values != nil { + val = item.Values + } + paramMap[item.Title] = val + } + } + + combinations := generateCombinations(paramMap) + + for _, cf := range tc.CustomFields { + if *cf.Id == automationTestNameID && cf.Value != nil { + for _, combo := range combinations { + results = append(results, gatheredCase{ + ID: caseID, + Title: *tc.Title, + Parameters: combo, + AutomationTestName: fmt.Sprintf("%v", *cf.Value), + }) + } + } else { + if automationTestNameID == 0 { + logrus.Infof("Custom field 'AutomationTestName' not found for %s-%d (%s)", projectID, tc.GetId(), tc.GetTitle()) + } + } + } + } + + if err := json.NewEncoder(os.Stdout).Encode(results); err != nil { + logrus.Fatalf("Failed to encode results to JSON: %v", err) + } +} + +// generateCombinations creates a Cartesian product of all parameter values. +// It takes a map where keys are parameter names and values are lists of possible values for that parameter. +// It returns a slice of maps, where each map represents a unique combination of parameter values. +func generateCombinations(params map[string][]string) []map[string]string { + // Extract keys to a slice to allow indexing during recursion. + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + + var results []map[string]string + var backtrack func(index int, current map[string]string) + + backtrack = func(index int, current map[string]string) { + // Base case: if we have processed all keys, we have a complete combination. + if index == len(keys) { + // Create a copy of the current map because 'current' is mutated in the recursion. + combo := make(map[string]string, len(current)) + for k, v := range current { + combo[k] = v + } + results = append(results, combo) + return + } + + key := keys[index] + values := params[key] + + // If a parameter has no values, skip it and proceed to the next key. + if len(values) == 0 { + backtrack(index+1, current) + } else { + // Iterate through each possible value for the current parameter. + for _, v := range values { + current[key] = v + backtrack(index+1, current) + // Backtrack: remove the key to restore the map state. + delete(current, key) + } + } + } + + backtrack(0, make(map[string]string)) + return results +} + func reportMetrics(params map[string]string) { logrus.Info("Granular reporting enabled.") // Granular reporting requires the metrics output file. @@ -194,7 +376,16 @@ func reportMetrics(params map[string]string) { // Report to Qase status := qase.StatusPassed - if !overallPass { + thresholdsFailed := false + for _, t := range thresholds { + if !t.Pass { + thresholdsFailed = true + break + } + } + if thresholdsFailed { + status = qase.StatusExceededThresholds + } else if !overallPass { status = qase.StatusFailed } diff --git a/qasereporter-k6/summary.go b/qase-k6-cli/summary.go similarity index 82% rename from qasereporter-k6/summary.go rename to qase-k6-cli/summary.go index 1d3c78dec..807108185 100644 --- a/qasereporter-k6/summary.go +++ b/qase-k6-cli/summary.go @@ -13,11 +13,14 @@ import ( const ( k6SummaryJsonFileEnvVar = "K6_SUMMARY_JSON_FILE" k6SummaryHtmlFileEnvVar = "K6_SUMMARY_HTML_FILE" + // See https://grafana.com/docs/k6/latest/results-output/web-dashboard/ + k6WebDashboardExportEnvVar = "K6_WEB_DASHBOARD_EXPORT" ) var ( - k6SummaryJsonFile = os.Getenv(k6SummaryJsonFileEnvVar) - k6SummaryHtmlFile = os.Getenv(k6SummaryHtmlFileEnvVar) + k6SummaryJsonFile = os.Getenv(k6SummaryJsonFileEnvVar) + k6SummaryHtmlFile = os.Getenv(k6SummaryHtmlFileEnvVar) + k6WebDashboardExportFile = os.Getenv(k6WebDashboardExportEnvVar) ) // K6Summary represents the structure of the k6 summary JSON output. @@ -37,13 +40,13 @@ type K6SummaryMetric struct { Thresholds map[string]K6SummaryThreshold `json:"thresholds,omitempty"` } -// K6SummaryThreshold represents a threshold with its pass/fail status. +// K6SummaryThreshold represents a threshold with its pass/fail status. type K6SummaryThreshold struct { OK bool `json:"ok"` } func reportSummary(params map[string]string) { - logrus.Info("Running k6 QASE reporter") + logrus.Info("Running qase-k6-cli reporter") if k6SummaryJsonFile == "" { logrus.Fatalf("Missing required environment variable: %s", k6SummaryJsonFileEnvVar) @@ -64,7 +67,16 @@ func reportSummary(params map[string]string) { // Report to Qase status := qase.StatusPassed - if !overallPass { + thresholdsFailed := false + for _, t := range thresholds { + if !t.Pass { + thresholdsFailed = true + break + } + } + if thresholdsFailed { + status = qase.StatusExceededThresholds + } else if !overallPass { status = qase.StatusFailed } @@ -80,6 +92,15 @@ func reportSummary(params map[string]string) { } } + if k6WebDashboardExportFile != "" { + if _, err := os.Stat(k6WebDashboardExportFile); err == nil { + logrus.Infof("Found Web Dashboard report at %s, preparing for upload.", k6WebDashboardExportFile) + attachments = append(attachments, k6WebDashboardExportFile) + } else { + logrus.Warnf("Web Dashboard report file specified but not found at %s.", k6WebDashboardExportFile) + } + } + logrus.Infof("Reporting to Qase: Project=%s, Run=%d, Case=%d, Status=%s", projectID, runID, testCaseID, status) // Upload attachments and get their hashes