diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6d3fcf2c..83ef746f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -26,3 +26,8 @@ updates: target-branch: "main" schedule: interval: "daily" + - package-ecosystem: "gradle" + directory: "/" + target-branch: "main" + schedule: + interval: "weekly" diff --git a/.github/workflows/agreements.yaml b/.github/workflows/agreements.yaml index 7547437b..572b02d8 100644 --- a/.github/workflows/agreements.yaml +++ b/.github/workflows/agreements.yaml @@ -7,7 +7,7 @@ on: jobs: call-workflow-agreements: - uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-agreements.yaml@v1.0.0 + uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-agreements.yaml@v1.2.1 secrets: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ secrets.PAT_CLATOOL }} diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml index d52dc247..8a4b051a 100644 --- a/.github/workflows/build-test-release.yml +++ b/.github/workflows/build-test-release.yml @@ -1,4 +1,3 @@ -# jscpd:ignore-start name: build-test-release on: push: @@ -8,1303 +7,20 @@ on: tags: - "v[0-9]+.[0-9]+.[0-9]+" pull_request: - branches: [main, develop] + branches: + - "**" jobs: - meta: - runs-on: ubuntu-latest - outputs: - sc4s: ghcr.io/${{ github.repository }}/container:${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} - container_tags: ${{ steps.docker_action_meta.outputs.tags }} - container_labels: ${{ steps.docker_action_meta.outputs.labels }} - container_buildtime: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.created'] }} - container_version: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} - container_revision: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.revision'] }} - container_base: ${{ fromJSON(steps.docker_action_meta.outputs.json).tags[0] }} - matrix_supportedSplunk: ${{ steps.matrix.outputs.supportedSplunk }} - matrix_supportedSC4S: ${{ steps.matrix.outputs.supportedSC4S }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: false - persist-credentials: false - - uses: actions/setup-node@v2 - with: - node-version: '14' - - name: Semantic Release - id: version - uses: cycjimmy/semantic-release-action@v2.6.0 - with: - semantic_version: 17 - extra_plugins: | - @semantic-release/exec - @semantic-release/git - semantic-release-helm - @google/semantic-release-replace-plugin - dry_run: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta - id: docker_action_meta - uses: docker/metadata-action@v3.6.0 - with: - images: ghcr.io/${{ github.repository }}/container - tags: | - type=sha,format=long - type=sha - type=semver,pattern={{version}},value=${{ steps.version.outputs.new_release_version }} - type=semver,pattern={{major}},value=${{ steps.version.outputs.new_release_version }} - type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.new_release_version }} - type=ref,event=branch - type=ref,event=pr - - name: matrix - id: matrix - uses: splunk/addonfactory-test-matrix-action@v1.3 - - compliance-sample-scanner: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: sample-scanner - uses: splunk/addonfactory-sample-scanner@v1 - id: sample-scanner - env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - compliance-dependencies: - name: compliance-dependencies - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: ort-action - uses: splunk/addonfactory-ort-action@v1 - id: ort-action - with: - WorkDir: . - UsePython3: "3.7" - - name: ort-action-artifacts-reports - uses: actions/upload-artifact@v2 - with: - name: analysis-reports - path: | - .ort/reports/* - if: always() - - name: ort-action-artifacts-analyzer - uses: actions/upload-artifact@v2 - with: - name: analysis-analyzer - path: | - .ort/analyzer/* - if: always() - compliance-copyrights: - name: compliance-copyrights - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: REUSE Compliance Check - uses: fsfe/reuse-action@v1.1 - - lint: - runs-on: ubuntu-latest - name: quality-lint - steps: - - uses: actions/checkout@v2 - with: - # Full git history is needed to get a proper list of changed files within `super-linter` - fetch-depth: 0 - - name: quality-lint - uses: docker://ghcr.io/github/super-linter:slim-v4 - env: - VALIDATE_ALL_CODEBASE: false - DEFAULT_BRANCH: main - LINTER_RULES_PATH: / - PYTHON_ISORT_CONFIG_FILE: pyproject.toml - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FILTER_REGEX_EXCLUDE: README.md - VALIDATE_GITHUB_ACTIONS: false - review_secrets: - name: security-detect-secrets - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: false - fetch-depth: "0" - - name: Trufflehog Actions Scan - uses: edplato/trufflehog-actions-scan@v0.9j-beta - with: - scanArguments: "--max_dept 50 -x .github/workflows/exclude-patterns.txt" - - semgrep: - runs-on: ubuntu-latest - name: security-sast-semgrep - steps: - - uses: actions/checkout@v2 - - name: Semgrep - id: semgrep - uses: returntocorp/semgrep-action@v1 - with: - publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} - - snyk: - name: security-vuln-snyk - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - uses: snyk/actions/setup@master - - uses: actions/setup-go@v2.1.4 - with: - go-version: "1.13" - - name: Snyk monitor - run: snyk test --sarif-file-output=snyk-scan_requirements.sarif --all-projects --print-deps --severity-threshold=high - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - - uses: actions/upload-artifact@v2 - if: always() - with: - name: snyk-results - path: snyk-scan_requirements.sarif - build: - name: build - runs-on: ubuntu-latest - needs: - - compliance-dependencies - outputs: - buildname: ${{ steps.buildupload.outputs.name }} - steps: - - uses: actions/checkout@v2 - with: - # Very Important semantic-release won't trigger a tagged - # build if this is not set false - persist-credentials: false - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - uses: actions/setup-node@v2 - with: - node-version: 14 - - name: create requirements file for pip - run: | - if [ -f "poetry.lock" ] - then - echo " potery.lock found " - sudo pip3 install poetry - mkdir -p package/lib || true - poetry export --without-hashes -o package/lib/requirements.txt - poetry export --without-hashes --dev -o requirements_dev.txt - cat requirements_dev.txt - fi - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Run Check there are libraries to scan - id: checklibs - run: if [ -f requirements_dev.txt ]; then echo "::set-output name=ENABLED::true"; fi - - name: pip cache - if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements_dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install deps - if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - run: pip install -r requirements_dev.txt - - name: Semantic Release Get Next - id: semantic - if: github.event_name != 'pull_request' - uses: cycjimmy/semantic-release-action@v2.6.0 - with: - semantic_version: 17 - extra_plugins: | - @semantic-release/exec - @semantic-release/git - dry_run: true - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} - - name: Determine the version to build - id: BuildVersion - uses: splunk/addonfactory-get-splunk-package-version-action@v1 - with: - SemVer: ${{ steps.semantic.outputs.new_release_version }} - PrNumber: ${{ github.event.number }} - - uses: actions/download-artifact@v2 - with: - name: analysis-reports - path: /tmp/analysis-reports - - name: Update Notices - run: | - cp -f /tmp/analysis-reports/NOTICE_default THIRDPARTY || true - cp -f /tmp/analysis-reports/NOTICE_default package/THIRDPARTY || true - - name: Build Package - id: uccgen - uses: splunk/addonfactory-ucc-generator-action@v1 - with: - version: ${{ steps.BuildVersion.outputs.VERSION }} - - name: Slim Package - id: slim - uses: splunk/addonfactory-packaging-toolkit-action@v1 - with: - source: ${{ steps.uccgen.outputs.OUTPUT }} - - name: artifact-splunk-unpacked - uses: actions/upload-artifact@v2 - with: - name: package-raw - path: ${{ steps.uccgen.outputs.OUTPUT }}** - if: always() - - name: artifact-splunk-base - uses: actions/upload-artifact@v2 - with: - name: package-splunkbase - path: ${{ steps.slim.outputs.OUTPUT }} - if: always() - - name: upload-build-to-s3 - id: buildupload - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - echo "::set-output name=name::$(basename "${{ steps.slim.outputs.OUTPUT }}")" - basename "${{ steps.slim.outputs.OUTPUT }}" - aws s3 cp "${{ steps.slim.outputs.OUTPUT }}" s3://ta-production-artifacts/ta-apps/ - - name: artifact-splunk-parts - uses: actions/upload-artifact@v2 - with: - name: package-deployment - path: build/package/deployment** - if: always() - security-virustotal: - name: security-virustotal - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v2 - with: - name: package-splunkbase - path: build/package/ - - - name: VirusTotal Scan - uses: crazy-max/ghaction-virustotal@v2 - with: - vt_api_key: ${{ secrets.VT_API_KEY }} - files: | - build/package/* - test-inventory: - runs-on: ubuntu-latest - # Map a step output to a job output - outputs: - unit: ${{ steps.testset.outputs.unit }} - knowledge: ${{ steps.testset.outputs.knowledge }} - ui: ${{ steps.testset.outputs.ui }} - modinput_functional: ${{ steps.testset.outputs.modinput_functional }} - requirement_test: ${{ steps.testset.outputs.requirement_test }} - steps: - - uses: actions/checkout@v2 - - id: testset - name: testsets - run: | - find tests -type d -maxdepth 1 -mindepth 1 | sed 's|^tests/||g' | while read -r TESTSET; do echo "::set-output name=$TESTSET::true"; echo "$TESTSET::true"; done - run-unit-tests: - name: test-unit-python3-${{ matrix.python-version }} - if: ${{ needs.test-inventory.outputs.unit == 'true' }} - runs-on: ubuntu-latest - needs: - - build - - test-inventory - strategy: - fail-fast: false - matrix: - python-version: - - "3.7" - steps: - - uses: actions/checkout@v2 - - if: failure() - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/download-artifact@v2 - with: - name: package-raw - path: output - - name: Setup addon - run: | - if [ -f "poetry.lock" ] - then - mkdir -p package/lib || true - pip install poetry - poetry export --without-hashes -o package/lib/requirements.txt - poetry export --without-hashes --dev -o requirements_dev.txt - fi - if [ ! -f requirements_dev.txt ]; then echo no requirements;exit 0 ;fi - pip install -r requirements_dev.txt - - name: Create directories - run: | - mkdir -p /opt/splunk/var/log/splunk - chmod -R 777 /opt/splunk/var/log/splunk - - name: Copy pytest ini - run: cp tests/unit/pytest-ci.ini pytest.ini - - name: Run Pytest with coverage - run: pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit - - name: Run Check if codecov enabled - id: checkcodecov - run: if [ -n "$CODECOV_TOKEN" ]; then echo "::set-output name=ENABLED::true"; fi - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Upload coverage to Codecov - if: ${{ steps.checkcodecov.outputs.ENABLED == 'true' }} - uses: codecov/codecov-action@v2.1.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - directory: ./coverage/reports/ - env_vars: OS,PYTHON - fail_ci_if_error: true - path_to_write_report: ./coverage/codecov_report.txt - verbose: true - - uses: actions/upload-artifact@v2 # upload test results - if: success() || failure() # run this step even if previous step failed - with: - name: test-results-unit-python_${{ matrix.python-version }} - path: test-results/* - - appinspect: - name: quality-appinspect-${{ matrix.tags }} - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - tags: - - "cloud" - - "appapproval" - - "deprecated_feature" - - "developer_guidance" - - "future" - - "self-service" - - "splunk_appinspect" - - "manual" - steps: - - name: Checkout for manual tag - if: matrix.tags == 'manual' - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: package-splunkbase - path: build/package/ - - name: Scan - uses: splunk/appinspect-cli-action@v1 - with: - app_path: build/package/ - included_tags: ${{ matrix.tags }} - - name: upload-appinspect-for-manual - if: always() && matrix.tags == 'manual' - uses: actions/upload-artifact@v2 - with: - name: appinspect_manual_checks.json - path: appinspect_result.json - - name: upload-manual-check-markdown - if: matrix.tags == 'manual' - uses: actions/upload-artifact@v2 - with: - name: manual_check_markdown.txt - path: manual_check_markdown.txt - - artifact-registry: - runs-on: ubuntu-latest - needs: - - security-virustotal - - meta - outputs: - artifact: ${{ steps.artifactid.outputs.result }} - steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: package-splunkbase - path: build/package/splunkbase - - - id: getappid - run: | - appid=$(jq -r '.info.id.name' package/app.manifest) - echo appid="$appid" - echo "::set-output name=result::$appid" - - run: | - curl -fsSL https://raw.githubusercontent.com/fishworks/gofish/main/scripts/install.sh | bash - gofish init - gofish install oras - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v1.10.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v3.6.0 - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=semver,pattern=v{{major}}.{{minor}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern=v{{major}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern=v{{version}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern={{major}}.{{minor}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern={{major}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern={{version}},prefix=${{ steps.getappid.outputs.result }}- - type=ref,event=branch,prefix=${{ steps.getappid.outputs.result }}- - type=ref,event=pr,prefix=${{ steps.getappid.outputs.result }}- - type=sha,prefix=${{ steps.getappid.outputs.result }}- - type=sha,format=long,prefix=${{ steps.getappid.outputs.result }}- - - name: Upload artifacts - run: | - tee /tmp/tags &>/dev/null <>$line<<" - oras push \ - --manifest-config /dev/null:application/vnd.splunk.ent.package.v1.tar+gzip \ - "$line" \ - "${{ steps.getappid.outputs.result }}".spl - echo " complete" - done < /tmp/tags - popd - - name: Output artifact locator - id: artifactid - run: | - echo "::set-output name=result:: ${{ needs.meta.outputs.sc4s }}" - - setup: - needs: - - build - - test-inventory - runs-on: ubuntu-18.04 - container: - image: ghcr.io/splunk/workflow-engine-base:2.0.3 - outputs: - argo-server: ${{ steps.test-setup.outputs.argo-server }} - argo-http1: ${{ steps.test-setup.outputs.argo-http1 }} - argo-secure: ${{ steps.test-setup.outputs.argo-secure }} - argo-href: "" - argo-base-href: ${{ steps.test-setup.outputs.argo-base-href }} - argo-workflow-tmpl-name: ${{ steps.test-setup.outputs.argo-workflow-tmpl-name }} - argo-namespace: ${{ steps.test-setup.outputs.argo-namespace }} - addon-name: ${{ steps.test-setup.outputs.addon-name }} - job-name: ${{ steps.test-setup.outputs.job-name }} - labels: ${{ steps.test-setup.outputs.labels }} - addon-upload-path: ${{ steps.test-setup.outputs.addon-upload-path }} - directory-path: ${{ steps.test-setup.outputs.directory-path }} - s3-bucket: ${{ steps.test-setup.outputs.s3-bucket }} - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: setup for test - id: test-setup - shell: bash - run: | - echo "::set-output name=argo-server::argo.wfe.splgdi.com:443" - echo "::set-output name=argo-http1::true" - echo "::set-output name=argo-secure::true" - echo "::set-output name=argo-base-href::\'\'" - echo "::set-output name=argo-namespace::workflows" - echo "::set-output name=argo-workflow-tmpl-name::ta-workflow" - - ADDON_NAME=$(crudini --get package/default/app.conf id name | tr '[:lower:]' '[:upper:]') - if [[ -n $(echo "${ADDON_NAME}" | awk -F 'SPLUNK_TA_' '{print $2}') ]]; - then - ADDON_NAME=$(echo "${ADDON_NAME}" | awk -F 'SPLUNK_TA_' '{print $2}') - elif [[ -n $(echo "${ADDON_NAME}" | awk -F '_FOR_SPLUNK' '{print $1}') ]]; - then - ADDON_NAME=$(echo "${ADDON_NAME}" | awk -F '_FOR_SPLUNK' '{print $1}') - fi - echo "::set-output name=addon-name::\"$ADDON_NAME\"" - - JOB_NAME=$(echo "$ADDON_NAME" | tail -c 16)-$(echo "${GITHUB_SHA}" | tail -c 8)-TEST-TYPE-${GITHUB_RUN_ID} - JOB_NAME=${JOB_NAME//[_.]/-} - echo "::set-output name=job-name::wf-$JOB_NAME" - - LABELS="addon-name=${ADDON_NAME}" - echo "::set-output name=labels::$LABELS" - - ADDON_UPLOAD_PATH="s3://ta-production-artifacts/ta-apps/${{ needs.build.outputs.buildname }}" - echo "::set-output name=addon-upload-path::$ADDON_UPLOAD_PATH" - echo "::set-output name=directory-path::/tmp" - echo "::set-output name=s3-bucket::ta-production-artifacts" - - run-knowledge-tests: - if: ${{ needs.test-inventory.outputs.knowledge == 'true' }} - needs: - - build - - test-inventory - - setup - - meta - runs-on: ubuntu-18.04 - strategy: - fail-fast: false - matrix: - splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} - sc4s: ${{ fromJson(needs.meta.outputs.matrix_supportedSC4S) }} - container: - image: ghcr.io/splunk/workflow-engine-base:2.0.3 - env: - ARGO_SERVER: ${{ needs.setup.outputs.argo-server }} - ARGO_HTTP1: ${{ needs.setup.outputs.argo-http1 }} - ARGO_SECURE: ${{ needs.setup.outputs.argo-secure }} - ARGO_BASE_HREF: ${{ needs.setup.outputs.argo-href }} - ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} - SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - TEST_TYPE: "knowledge" - TEST_ARGS: "" - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Read secrets from AWS Secrets Manager into environment variables - id: get-argo-token - run: | - ARGO_TOKEN=$(aws secretsmanager get-secret-value --secret-id ta-github-workflow-automation-token | jq -r '.SecretString') - echo "::set-output name=argo-token::$ARGO_TOKEN" - - name: create job name - id: create-job-name - shell: bash - run: | - RANDOM_STRING=$(head -3 /dev/urandom | tr -cd '[:lower:]' | cut -c -4) - JOB_NAME=${{ needs.setup.outputs.job-name }}-${RANDOM_STRING} - JOB_NAME=${JOB_NAME//TEST-TYPE/${{ env.TEST_TYPE }}} - JOB_NAME=${JOB_NAME//[_.]/-} - JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]') - echo "::set-output name=job-name::$JOB_NAME" - - - name: run-tests - id: run-tests - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v1.4.6 - with: - splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - test-type: ${{ env.TEST_TYPE }} - test-args: "" - job-name: ${{ steps.create-job-name.outputs.job-name }} - labels: ${{ needs.setup.outputs.labels }} - workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }} - workflow-template-ns: ${{ needs.setup.outputs.argo-namespace }} - delay-destroy: "No" - addon-url: ${{ needs.setup.outputs.addon-upload-path }} - addon-name: ${{ needs.setup.outputs.addon-name }} - sc4s-version: ${{ matrix.sc4s.version }} - - name: Check if pod was deleted - id: is-pod-deleted - if: always() - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - run: | - set -o xtrace - if argo watch ${{ steps.run-tests.outputs.workflow-name }} -n workflows | grep "pod deleted"; then - echo "::set-output name=retry-workflow::true" - fi - - name: Retrying workflow - id: retry-wf - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - if: always() - run: | - set -o xtrace - set +e - if [[ "${{ steps.is-pod-deleted.outputs.retry-workflow }}" == "true" ]] - then - WORKFLOW_NAME=$(argo resubmit -v -o json -n workflows "${{ steps.run-tests.outputs.workflow-name }}" | jq -r .metadata.name) - echo "::set-output name=workflow-name::$WORKFLOW_NAME" - argo logs --follow "${WORKFLOW_NAME}" -n workflows || echo "... there was an error fetching logs, the workflow is still in progress. please wait for the workflow to complete ..." - else - echo "No retry required" - argo wait "${{ steps.run-tests.outputs.workflow-name }}" -n workflows - argo watch "${{ steps.run-tests.outputs.workflow-name }}" -n workflows | grep "test-addon" - fi - - name: check if workflow completed - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - shell: bash - if: always() - run: | - set +e - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - echo "Status of workflow:" "$ARGO_STATUS" - while [ "$ARGO_STATUS" == "Running" ] || [ "$ARGO_STATUS" == "Pending" ] - do - echo "... argo Workflow ${WORKFLOW_NAME} is running, waiting for it to complete." - argo wait "${WORKFLOW_NAME}" -n workflows || true - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - done - - name: pull artifacts from s3 bucket - if: always() - run: | - echo "pulling artifacts" - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/artifacts-${{ steps.create-job-name.outputs.job-name }}/${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - tar -xf ${{ needs.setup.outputs.directory-path }}/${{ steps.create-job-name.outputs.job-name }}.tgz -C ${{ needs.setup.outputs.directory-path }} - - name: pull logs from s3 bucket - if: always() - run: | - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - echo "pulling logs" - mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests artifacts - path: | - ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests logs - path: | - ${{ needs.setup.outputs.directory-path }}/argo-logs - - name: Upload cim-compliance-report for ${{ matrix.splunk.version }} - uses: actions/upload-artifact@v2 - if: ${{ matrix.splunk.islatest == true }} - with: - name: cim-compliance-report - path: | - ${{ needs.setup.outputs.directory-path }}/test-results/cim-compliance-report.md - - name: Upload cim-field-report for ${{ matrix.splunk.version }} - uses: actions/upload-artifact@v2 - if: ${{ matrix.splunk.islatest == true }} - with: - name: cim-field-report - path: | - ${{ needs.setup.outputs.directory-path }}/test-results/cim_field_report.json - - name: Test Report - uses: dorny/test-reporter@v1 - if: always() - with: - name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} test report - path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" - reporter: java-junit - run-requirement-tests: - if: ${{ needs.test-inventory.outputs.requirement_test == 'true' }} - needs: - - build - - test-inventory - - setup - - meta - runs-on: ubuntu-18.04 - strategy: - fail-fast: false - matrix: - splunk: ['8.2.1'] - sc4s: ${{ fromJson(needs.meta.outputs.matrix_supportedSC4S) }} - container: - image: ghcr.io/splunk/workflow-engine-base:2.0.3 - env: - ARGO_SERVER: ${{ needs.setup.outputs.argo-server }} - ARGO_HTTP1: ${{ needs.setup.outputs.argo-http1 }} - ARGO_SECURE: ${{ needs.setup.outputs.argo-secure }} - ARGO_BASE_HREF: ${{ needs.setup.outputs.argo-href }} - ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} - TEST_TYPE: "requirement_test" - TEST_ARGS: "" - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Read secrets from AWS Secrets Manager into environment variables - id: get-argo-token - run: | - ARGO_TOKEN=$(aws secretsmanager get-secret-value --secret-id ta-github-workflow-automation-token | jq -r '.SecretString') - echo "::set-output name=argo-token::$ARGO_TOKEN" - - name: create job name - id: create-job-name - shell: bash - run: | - RANDOM_STRING=$(head -3 /dev/urandom | tr -cd '[:lower:]' | cut -c -4) - JOB_NAME=${{ needs.setup.outputs.job-name }}-${RANDOM_STRING} - JOB_NAME=${JOB_NAME//TEST-TYPE/${{ env.TEST_TYPE }}} - JOB_NAME=${JOB_NAME//[_.]/-} - JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]') - echo "::set-output name=job-name::$JOB_NAME" - - - name: run-tests - id: run-tests - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v1.4.6 - with: - splunk: ${{ matrix.splunk }} - test-type: ${{ env.TEST_TYPE }} - test-args: "" - job-name: ${{ steps.create-job-name.outputs.job-name }} - labels: ${{ needs.setup.outputs.labels }} - workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }} - workflow-template-ns: ${{ needs.setup.outputs.argo-namespace }} - delay-destroy: "No" - addon-url: ${{ needs.setup.outputs.addon-upload-path }} - addon-name: ${{ needs.setup.outputs.addon-name }} - sc4s-version: ${{ matrix.sc4s.version }} - - name: Check if pod was deleted - id: is-pod-deleted - if: always() - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - run: | - set -o xtrace - if argo watch ${{ steps.run-tests.outputs.workflow-name }} -n workflows | grep "pod deleted"; then - echo "::set-output name=retry-workflow::true" - fi - - name: Retrying workflow - id: retry-wf - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - if: always() - run: | - set -o xtrace - set +e - if [[ "${{ steps.is-pod-deleted.outputs.retry-workflow }}" == "true" ]] - then - WORKFLOW_NAME=$(argo resubmit -v -o json -n workflows "${{ steps.run-tests.outputs.workflow-name }}" | jq -r .metadata.name) - echo "::set-output name=workflow-name::$$WORKFLOW_NAME" - argo logs --follow "${WORKFLOW_NAME}" -n workflows || echo "... there was an error fetching logs, the workflow is still in progress. please wait for the workflow to complete ..." - else - echo "No retry required" - argo wait "${{ steps.run-tests.outputs.workflow-name }}" -n workflows - argo watch "${{ steps.run-tests.outputs.workflow-name }}" -n workflows | grep "test-addon" - fi - - name: check if workflow completed - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - shell: bash - if: always() - run: | - set +e - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - echo "Status of workflow:" "$ARGO_STATUS" - while [ "$ARGO_STATUS" == "Running" ] || [ "$ARGO_STATUS" == "Pending" ] - do - echo "... argo Workflow ${WORKFLOW_NAME} is running, waiting for it to complete." - argo wait "${WORKFLOW_NAME}" -n workflows || true - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - done - - name: pull artifacts from s3 bucket - if: always() - run: | - echo "pulling artifacts" - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/artifacts-${{ steps.create-job-name.outputs.job-name }}/${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - tar -xf ${{ needs.setup.outputs.directory-path }}/${{ steps.create-job-name.outputs.job-name }}.tgz -C ${{ needs.setup.outputs.directory-path }} - - name: pull logs from s3 bucket - if: always() - run: | - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - echo "pulling logs" - mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk }} ${{ env.TEST_TYPE }} tests artifacts - path: | - ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk }} ${{ env.TEST_TYPE }} tests logs - path: | - ${{ needs.setup.outputs.directory-path }}/argo-logs - - name: Test Report - uses: dorny/test-reporter@v1 - if: always() - with: - name: splunk ${{ matrix.splunk }} ${{ env.TEST_TYPE }} test report - path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" - reporter: java-junit - run-ui-tests: - if: ${{ needs.test-inventory.outputs.ui == 'true' }} - needs: - - build - - test-inventory - - setup - - meta - runs-on: ubuntu-18.04 - strategy: - fail-fast: false - matrix: - splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} - browser: ["chrome"] - container: - image: ghcr.io/splunk/workflow-engine-base:2.0.3 - env: - ARGO_SERVER: ${{ needs.setup.outputs.argo-server }} - ARGO_HTTP1: ${{ needs.setup.outputs.argo-http1 }} - ARGO_SECURE: ${{ needs.setup.outputs.argo-secure }} - ARGO_BASE_HREF: ${{ needs.setup.outputs.argo-href }} - ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} - SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - TEST_TYPE: "ui" - TEST_ARGS: "--browser ${{ matrix.browser }}" - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Read secrets from AWS Secrets Manager into environment variables - id: get-argo-token - run: | - ARGO_TOKEN=$(aws secretsmanager get-secret-value --secret-id ta-github-workflow-automation-token | jq -r '.SecretString') - echo "::set-output name=argo-token::$ARGO_TOKEN" - - name: create job name - id: create-job-name - shell: bash - run: | - RANDOM_STRING=$(head -3 /dev/urandom | tr -cd '[:lower:]' | cut -c -4) - JOB_NAME=${{ needs.setup.outputs.job-name }}-${RANDOM_STRING} - JOB_NAME=${JOB_NAME//TEST-TYPE/${{ env.TEST_TYPE }}-${{ matrix.browser }}} - JOB_NAME=${JOB_NAME//[_.:]/-} - JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]') - echo "::set-output name=job-name::$JOB_NAME" - - name: run-tests - id: run-tests - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v1.4.6 - with: - splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - test-type: ${{ env.TEST_TYPE }} - test-args: ${{ env.TEST_ARGS }} - job-name: ${{ steps.create-job-name.outputs.job-name }} - labels: ${{ needs.setup.outputs.labels }} - workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }} - workflow-template-ns: ${{ needs.setup.outputs.argo-namespace }} - delay-destroy: "No" - addon-url: ${{ needs.setup.outputs.addon-upload-path }} - addon-name: ${{ needs.setup.outputs.addon-name }} - sc4s-version: "No" - - name: Check if pod was deleted - id: is-pod-deleted - if: always() - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - run: | - set -o xtrace - if argo watch ${{ steps.run-tests.outputs.workflow-name }} -n workflows | grep "pod deleted" ; then - echo "::set-output name=retry-workflow::true" - fi - - name: Retrying workflow - id: retry-wf - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - if: always() - run: | - set -o xtrace - set +e - if [[ "${{ steps.is-pod-deleted.outputs.retry-workflow }}" == "true" ]] - then - WORKFLOW_NAME=$(argo resubmit -v -o json -n workflows "${{ steps.run-tests.outputs.workflow-name }}" | jq -r .metadata.name) - echo "::set-output name=workflow-name::$WORKFLOW_NAME" - argo logs --follow "${WORKFLOW_NAME}" -n workflows || echo "... there was an error fetching logs, the workflow is still in progress. please wait for the workflow to complete ..." - else - echo "No retry required" - argo wait "${{ steps.run-tests.outputs.workflow-name }}" -n workflows - argo watch "${{ steps.run-tests.outputs.workflow-name }}" -n workflows | grep "test-addon" - fi - - name: check if workflow completed - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - if: always() - shell: bash - run: | - set +e - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - echo "Status of workflow:" "$ARGO_STATUS" - while [ "$ARGO_STATUS" == "Running" ] || [ "$ARGO_STATUS" == "Pending" ] - do - echo "... argo Workflow ${WORKFLOW_NAME} is running, waiting for it to complete." - argo wait "${WORKFLOW_NAME}" -n workflows || true - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - done - - name: pull artifacts from s3 bucket - if: always() - run: | - echo "pulling artifacts" - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/artifacts-${{ steps.create-job-name.outputs.job-name }}/${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - tar -xf ${{ needs.setup.outputs.directory-path }}/${{ steps.create-job-name.outputs.job-name }}.tgz -C ${{ needs.setup.outputs.directory-path }} - - name: pull logs from s3 bucket - if: always() - run: | - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - echo "pulling logs" - mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests artifacts - path: | - ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} tests logs - path: | - ${{ needs.setup.outputs.directory-path }}/argo-logs - - name: Test Report - uses: dorny/test-reporter@v1 - if: always() - with: - name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} test report - path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" - reporter: java-junit - run-modinput-tests: - if: ${{ needs.test-inventory.outputs.modinput_functional == 'true' }} - needs: - - build - - test-inventory - - setup - - meta - runs-on: ubuntu-18.04 - strategy: - fail-fast: false - matrix: - splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} - modinput-type: ["modinput_functional"] - container: - image: ghcr.io/splunk/workflow-engine-base:2.0.3 - env: - ARGO_SERVER: ${{ needs.setup.outputs.argo-server }} - ARGO_HTTP1: ${{ needs.setup.outputs.argo-http1 }} - ARGO_SECURE: ${{ needs.setup.outputs.argo-secure }} - ARGO_BASE_HREF: ${{ needs.setup.outputs.argo-href }} - ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} - SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - TEST_TYPE: "modinput_functional" - TEST_ARGS: "" - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Read secrets from AWS Secrets Manager into environment variables - id: get-argo-token - run: | - ARGO_TOKEN=$(aws secretsmanager get-secret-value --secret-id ta-github-workflow-automation-token | jq -r '.SecretString') - echo "::set-output name=argo-token::$ARGO_TOKEN" - - name: create job name - id: create-job-name - shell: bash - run: | - RANDOM_STRING=$(head -3 /dev/urandom | tr -cd '[:lower:]' | cut -c -4) - JOB_NAME=${{ needs.setup.outputs.job-name }}-${RANDOM_STRING} - JOB_NAME=${JOB_NAME//TEST-TYPE/${{ env.TEST_TYPE }}} - JOB_NAME=${JOB_NAME//[_.]/-} - JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]') - echo "::set-output name=job-name::$JOB_NAME" - - name: run-tests - id: run-tests - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v1.4.6 - with: - splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} - test-type: ${{ env.TEST_TYPE }} - test-args: ${{ env.TEST_ARGS }} - job-name: ${{ steps.create-job-name.outputs.job-name }} - labels: ${{ needs.setup.outputs.labels }} - workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }} - workflow-template-ns: ${{ needs.setup.outputs.argo-namespace }} - delay-destroy: "No" - addon-url: ${{ needs.setup.outputs.addon-upload-path }} - addon-name: ${{ needs.setup.outputs.addon-name }} - sc4s-version: "No" - - name: Check if pod was deleted - id: is-pod-deleted - if: always() - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - run: | - set -o xtrace - if argo watch ${{ steps.run-tests.outputs.workflow-name }} -n workflows | grep "pod deleted"; then - echo "::set-output name=retry-workflow::true" - fi - - name: Retrying workflow - id: retry-wf - shell: bash - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - if: always() - run: | - set -o xtrace - set +e - if [[ "${{ steps.is-pod-deleted.outputs.retry-workflow }}" == "true" ]] - then - WORKFLOW_NAME=$(argo resubmit -v -o json -n workflows "${{ steps.run-tests.outputs.workflow-name }}" | jq -r .metadata.name) - echo "::set-output name=workflow-name::$WORKFLOW_NAME" - argo logs --follow "${WORKFLOW_NAME}" -n workflows || echo "... there was an error fetching logs, the workflow is still in progress. please wait for the workflow to complete ..." - else - echo "No retry required" - argo wait "${{ steps.run-tests.outputs.workflow-name }}" -n workflows - argo watch "${{ steps.run-tests.outputs.workflow-name }}" -n workflows | grep "test-addon" - fi - - name: check if workflow completed - env: - ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - if: always() - shell: bash - run: | - set +e - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - echo "Status of workflow:" "$ARGO_STATUS" - while [ "$ARGO_STATUS" == "Running" ] || [ "$ARGO_STATUS" == "Pending" ] - do - echo "... argo Workflow ${WORKFLOW_NAME} is running, waiting for it to complete." - argo wait "${WORKFLOW_NAME}" -n workflows || true - ARGO_STATUS=$(argo get "${WORKFLOW_NAME}" -n workflows -o json | jq -r '.status.phase') - done - - name: pull artifacts from s3 bucket - if: always() - run: | - echo "pulling artifacts" - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/artifacts-${{ steps.create-job-name.outputs.job-name }}/${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - tar -xf ${{ needs.setup.outputs.directory-path }}/${{ steps.create-job-name.outputs.job-name }}.tgz -C ${{ needs.setup.outputs.directory-path }} - - name: pull logs from s3 bucket - if: always() - run: | - # shellcheck disable=SC2157 - if [ -z "${{ steps.retry-wf.outputs.workflow-name }}" ]; then - WORKFLOW_NAME=${{ steps.run-tests.outputs.workflow-name }} - else - WORKFLOW_NAME="${{ steps.retry-wf.outputs.workflow-name }}" - fi - echo "pulling logs" - mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs - aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests artifacts - path: | - ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v2.2.4 - if: always() - with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests logs - path: | - ${{ needs.setup.outputs.directory-path }}/argo-logs - - name: Test Report - uses: dorny/test-reporter@v1 - if: always() - with: - name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} test report - path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" - reporter: java-junit - - pre-publish: - if: always() - needs: - - meta - - compliance-sample-scanner - - compliance-dependencies - - compliance-copyrights - - lint - - review_secrets - - semgrep - - snyk - - build - - security-virustotal - - test-inventory - - run-unit-tests - - appinspect - - setup - - run-knowledge-tests - - run-modinput-tests - - run-ui-tests - - run-requirement-tests - runs-on: ubuntu-latest - env: - NEEDS: ${{ toJson(needs) }} - steps: - - name: check if tests have passed or skipped - id: check - shell: bash - run: | - RUN_PUBLISH=$(echo "$NEEDS" | jq ".[] | select( ( .result != \"skipped\" ) and .result != \"success\" ) | length == 0") - if [[ "$RUN_PUBLISH" != *'false'* ]] - then - echo "::set-output name=run-publish::true" - else - echo "::set-output name=run-publish::false" - fi - - name: exit without publish - if: ${{ steps.check.outputs.run-publish == 'false' }} - run: | - echo " some test job failed. " - exit 1 - - publish: - if: always() && needs.pre-publish.result == 'success' && github.event_name != 'pull_request' - needs: - - pre-publish - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: false - persist-credentials: false - - uses: actions/setup-node@v2 - with: - node-version: '14' - - name: Semantic Release - id: semantic - uses: cycjimmy/semantic-release-action@v2.6.0 - with: - semantic_version: 17 - extra_plugins: | - @semantic-release/exec - @semantic-release/git - @semantic-release/commit-analyzer - @semantic-release/release-notes-generator - @semantic-release/github - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} - - name: Download package-deployment - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} - uses: actions/download-artifact@v2 - id: download-package-deployment - with: - name: package-deployment - path: download/artifacts/ - - name: Download package-splunkbase - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} - uses: actions/download-artifact@v2 - id: download-package-splunkbase - with: - name: package-splunkbase - path: download/artifacts/deployment - - name: Download cim-compliance-report - id: download-cim-compliance-report - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} - continue-on-error: true - uses: actions/download-artifact@v2 - with: - name: cim-compliance-report - path: download/artifacts/deployment - - name: Download cim-field-report - id: download-cim-field-report - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} - continue-on-error: true - uses: actions/download-artifact@v2 - with: - name: cim-field-report - path: download/artifacts/deployment - - name: List of assets - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} - run: | - ls -la ${{ steps.download-package-splunkbase.outputs.download-path }} - - name: Upload assets to release - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GH_TOKEN_ADMIN }} - file: ${{ steps.download-package-splunkbase.outputs.download-path }}/* - overwrite: true - file_glob: true - tag: v${{ steps.semantic.outputs.new_release_version }} - - update-semver: - name: release-set-git-tags - if: always() && needs.publish.result == 'success' && startsWith(github.ref, 'refs/tags/v') - needs: - - publish - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: haya14busa/action-update-semver@v1 + call-workflow: + uses: splunk/github-workflow-build-test-release/.github/workflows/reusable-build-test-release.yml@v1.0.1 + secrets: + GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SEMGREP_PUBLISH_TOKEN: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + VT_API_KEY: ${{ secrets.VT_API_KEY }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + OTHER_TA_REQUIRED_CONFIGS: ${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} diff --git a/.github/workflows/publish-manual.yaml b/.github/workflows/publish-manual.yaml index 94cea5ef..cc08856f 100644 --- a/.github/workflows/publish-manual.yaml +++ b/.github/workflows/publish-manual.yaml @@ -7,145 +7,8 @@ on: required: true jobs: - - validate_branch: - name: validate_branch - runs-on: ubuntu-latest - steps: - - name: validate_branch - if: github.ref != 'refs/heads/develop' && github.ref != 'refs/heads/main' - run: | - echo "Skipping publish as branch is neither main nor develop" - - build: - needs: - - validate_branch - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' - name: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - # Very Important semantic-release won't trigger a tagged - # build if this is not set false - persist-credentials: false - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - uses: actions/setup-node@v2 - with: - node-version: 14 - - name: create requirements file for pip - run: | - if [ -f "poetry.lock" ] - then - echo " potery.lock found " - sudo pip3 install poetry - mkdir -p package/lib || true - poetry export --without-hashes -o package/lib/requirements.txt - poetry export --without-hashes --dev -o requirements_dev.txt - cat requirements_dev.txt - fi - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Run Check there are libraries to scan - id: checklibs - run: if [ -f requirements_dev.txt ]; then echo "::set-output name=ENABLED::true"; fi - - name: pip cache - if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements_dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install deps - if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - run: pip install -r requirements_dev.txt - - name: Determine the version to build - id: BuildVersion - uses: splunk/addonfactory-get-splunk-package-version-action@v1 - with: - SemVer: ${{ github.event.inputs.tag }} - PrNumber: ${{ github.event.number }} - - name: Build Package - id: uccgen - uses: splunk/addonfactory-ucc-generator-action@v1 - with: - version: ${{ steps.BuildVersion.outputs.VERSION }} - - name: Slim Package - id: slim - uses: splunk/addonfactory-packaging-toolkit-action@v1 - with: - source: ${{ steps.uccgen.outputs.OUTPUT }} - - name: artifact-splunk-unpacked - uses: actions/upload-artifact@v2 - with: - name: package-raw - path: ${{ steps.uccgen.outputs.OUTPUT }}** - if: always() - - name: artifact-splunk-base - uses: actions/upload-artifact@v2 - with: - name: package-splunkbase - path: ${{ steps.slim.outputs.OUTPUT }} - if: always() - - name: artifact-splunk-parts - uses: actions/upload-artifact@v2 - with: - name: package-deployment - path: build/package/deployment** - if: always() - - publish: - needs: - - build - - validate_branch - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: false - persist-credentials: false - - name: Release on github - if: github.ref == 'refs/heads/main' - uses: "marvinpinto/action-automatic-releases@latest" - with: - repo_token: "${{ secrets.GITHUB_TOKEN }}" - prerelease: false - automatic_release_tag: v${{ github.event.inputs.tag }} - - name: Release on github (pre-release) - if: github.ref == 'refs/heads/develop' - uses: "marvinpinto/action-automatic-releases@latest" - with: - repo_token: "${{ secrets.GITHUB_TOKEN }}" - prerelease: true - automatic_release_tag: v${{ github.event.inputs.tag }} - - name: Download package-deployment - uses: actions/download-artifact@v2 - id: download-package-deployment - with: - name: package-deployment - path: download/artifacts/ - - name: Download package-splunkbase - uses: actions/download-artifact@v2 - id: download-package-splunkbase - with: - name: package-splunkbase - path: download/artifacts/deployment - - name: List of assets - run: | - ls -la ${{ steps.download-package-splunkbase.outputs.download-path }} - - name: Upload assets to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GH_TOKEN_ADMIN }} - file: ${{ steps.download-package-splunkbase.outputs.download-path }}/* - overwrite: true - file_glob: true - tag: v${{ github.event.inputs.tag }} \ No newline at end of file + call-workflow: + uses: splunk/github-workflow-publish-manual/.github/workflows/reusable-publish-manual.yaml@v1.0.1 + secrets: + GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-notes.yaml b/.github/workflows/release-notes.yaml index 7e6ff50d..dfbdeafc 100644 --- a/.github/workflows/release-notes.yaml +++ b/.github/workflows/release-notes.yaml @@ -7,6 +7,6 @@ on: jobs: call-workflow-preview: - uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-release-notes.yaml@v1.0.0 + uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-release-notes.yaml@v1.2.1 secrets: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.slimignore b/.slimignore index f4658c81..e43b0f98 100644 --- a/.slimignore +++ b/.slimignore @@ -1,3 +1 @@ .DS_Store -eventgen.conf -samples \ No newline at end of file