ci: add suitespec to GitLab [backport 2.17] (#11419)

Backport f7c102e from #10877 to 2.17. We port the suitespec machinery to GitLab. The current implementation uses generated child pipelines, with the tests YAML configuration automatically generated to include the required jobs based on the files listed in the PR. The job information previously contained in the config.templ.yaml file has been migrated to the jobspec.yml configuration. The jobspec can be "modular", in the sense that it can be split into multiple jobspec.yml file within the `tests/` subtree (e.g. `tests/jobspec.yml`, `tests/debugging/jobspec.yml`, ...) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) Co-authored-by: Gabriele N. Tornetta <[email protected]> Co-authored-by: erikayasuda <[email protected]>
DataDog · Nov 22, 2024 · f3c25f7 · f3c25f7
1 parent 475d911
commit f3c25f7
Show file tree

Hide file tree

Showing 29 changed files with 929 additions and 721 deletions.
diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml
@@ -4,7 +4,7 @@ default_resource_class: &default_resource_class medium
 ubuntu_base_image: &ubuntu_base_img ubuntu-2004:2023.04.2
 cimg_base_image: &cimg_base_image cimg/base:2022.08
 python310_image: &python310_image cimg/python:3.10.12
-ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
+ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:8ca43d46ff34e078bd7bc0662e74e6be38547a98140a5cd4203805f6b214b583
 redis_image: &redis_image redis:4.0-alpine@sha256:3e99741f293147ff406657dda7644c2b88564b80a498cd00da8f905743449c9f
 memcached_image: &memcached_image memcached:1.5-alpine@sha256:48cb7207e3d34871893fa1628f3a4984375153e9942facf82e25935b0a633c8a
 cassandra_image: &cassandra_image cassandra:3.11.7@sha256:495e5752526f7e75d3ad85b6a6bbf3b79714321b17a44255a216c341e3baae11

diff --git a/.github/workflows/requirements-locks.yml b/.github/workflows/requirements-locks.yml
@@ -11,7 +11,7 @@ jobs:
   validate:
     name: Check requirements lockfiles
     runs-on: ubuntu-latest
-    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
+    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:8ca43d46ff34e078bd7bc0662e74e6be38547a98140a5cd4203805f6b214b583
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/.gitignore b/.gitignore
@@ -162,6 +162,8 @@ template_venv/
 cloned_venvs/
 # CircleCI generated config
 .circleci/config.gen.yml
+# GitLab CI generated config
+.gitlab/*-gen.yml
 
 # Automatically generated fixtures
 tests/appsec/iast/fixtures/aspects/callers.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 stages:
   - package
-  - tests
+  - tests-gen
+  - tests-trigger
   - shared-pipeline
   - dogfood
   - benchmarks
@@ -10,27 +11,37 @@ stages:
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
-  TESTRUNNER_IMAGE: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:ecc5741ff3e7c8a30363fcd9cca79a371dcea5b4
   # CI_DEBUG_SERVICES: "true"
 
-.testrunner:
-  image: $TESTRUNNER_IMAGE
-  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
-  tags: [ "arch:amd64" ]
-  timeout: 20m
-  before_script:
-    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
-    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
-
 include:
   - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml
   - local: ".gitlab/services.yml" # Include early so others can use the definitions
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/package.yml"
-  - local: ".gitlab/tests.yml"
   - local: ".gitlab/macrobenchmarks.yml"
   - local: ".gitlab/dogfood.yml"
   - local: ".gitlab/release.yml"
+  - local: ".gitlab/testrunner.yml"
+
+tests-gen:
+  stage: tests-gen
+  extends: .testrunner
+  script:
+    - pip install riot==0.20.0
+    - riot -v run --pass-env -s gitlab-gen-config -v
+  needs: []
+  artifacts:
+    paths:
+      - .gitlab/tests-gen.yml
+
+run-tests-trigger:
+ stage: tests-trigger
+ needs: [ tests-gen ]
+ trigger:
+   include:
+     - artifact: .gitlab/tests-gen.yml
+       job: tests-gen
+   strategy: depend
 
 requirements_json_test:
   rules:

diff --git a/.gitlab/package.yml b/.gitlab/package.yml
@@ -1,24 +1,3 @@
-build_base_venvs:
-  extends: .testrunner
-  stage: package
-  parallel:
-    matrix:
-      - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
-  variables:
-    CMAKE_BUILD_PARALLEL_LEVEL: 12
-    PIP_VERBOSE: 1
-    DD_PROFILING_NATIVE_TESTS: 1
-  script:
-    - pip install riot==0.20.0
-    - riot -P -v generate --python=$PYTHON_VERSION
-  artifacts:
-    name: venv_$PYTHON_VERSION
-    paths:
-      - .riot/venv_*
-      - ddtrace/**/*.so*
-      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
-      - ddtrace/internal/datadog/profiling/test/test_*
-
 download_ddtrace_artifacts:
   image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
   tags: [ "arch:amd64" ]

diff --git a/.gitlab/testrunner.yml b/.gitlab/testrunner.yml
@@ -0,0 +1,8 @@
+.testrunner:
+  image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:7a2e802af76051f82d698919d2837eff18dbb48e
+  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
+  tags: [ "arch:amd64" ]
+  timeout: 20m
+  before_script:
+    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
+    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml
@@ -1,5 +1,23 @@
+stages:
+  - tests
+
 variables:
   RIOT_RUN_CMD: riot -P -v run --exitfirst --pass-env -s
+  REPO_LANG: python # "python" is used everywhere rather than "py"
+  # CI_DEBUG_SERVICES: "true"
+
+
+.testrunner:
+  image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:7a2e802af76051f82d698919d2837eff18dbb48e
+  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
+  tags: [ "arch:amd64" ]
+  timeout: 20m
+  before_script:
+    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
+    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
+
+
+{{services.yml}}
 
 .test_base_hatch:
   extends: .testrunner
@@ -22,6 +40,7 @@ variables:
         hatch run ${env}:test
       done
 
+
 .test_base_hatch_snapshot:
   extends: .test_base_hatch
   services:
@@ -32,6 +51,28 @@ variables:
     # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent
     - export DD_TRACE_AGENT_URL="http://testagent:9126"
 
+
+build_base_venvs:
+  extends: .testrunner
+  stage: tests
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+  variables:
+    CMAKE_BUILD_PARALLEL_LEVEL: 12
+    PIP_VERBOSE: 1
+    DD_PROFILING_NATIVE_TESTS: 1
+  script:
+    - pip install riot==0.20.0
+    - riot -P -v generate --python=$PYTHON_VERSION
+  artifacts:
+    name: venv_$PYTHON_VERSION
+    paths:
+      - .riot/venv_*
+      - ddtrace/**/*.so*
+      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
+      - ddtrace/internal/datadog/profiling/test/test_*
+
 .test_base_riot:
   extends: .testrunner
   stage: tests
@@ -71,13 +112,20 @@ variables:
     # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the
     # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent
     - export DD_TRACE_AGENT_URL="http://testagent:9126"
+    - ln -s "${CI_PROJECT_DIR}" "/root/project"
+
+slotscheck:
+  extends: .testrunner
+  stage: tests
+  needs: []
+  script:
+    - hatch run slotscheck:_
+
+conftests:
+  extends: .testrunner
+  stage: tests
+  needs: []
+  script:
+    - hatch run meta-testing:meta-testing
 
-include:
-  - local: ".gitlab/tests/appsec.yml"
-  - local: ".gitlab/tests/ci_visibility.yml"
-  - local: ".gitlab/tests/contrib.yml"
-  - local: ".gitlab/tests/core.yml"
-  - local: ".gitlab/tests/debugging.yml"
-  - local: ".gitlab/tests/llmobs.yml"
-  - local: ".gitlab/tests/tracer.yml"
-  - local: ".gitlab/tests/profiling.yml"
+# Required jobs will appear here
diff --git a/.gitlab/tests/appsec.yml b/.gitlab/tests/appsec.yml
diff --git a/.gitlab/tests/ci_visibility.yml b/.gitlab/tests/ci_visibility.yml