Onboarding: New aws account

.github/CODEOWNERS

@@ -30,4 +30,4 @@
 /manifests/ruby.yml @DataDog/ruby-guild @DataDog/asm-ruby
 
 # Allows everyone to easily make changes
-/tests/telemetry_intake/static/ @DataDog/apm-ecosystems
+/tests/telemetry_intake/static/ @DataDog/apm-sdk

.github/actions/get_target_branch/action.yaml

@@ -0,0 +1,25 @@
+name: Get_target_branch
+description: "Gets target branch from the PR description"
+inputs:
+  text:
+    description: "Text from which to extract the target branch"
+    required: true
+outputs:
+  target-branch:
+    description: "Target branch"
+    value: ${{ steps.extract.outputs.target-branch }}
+
+runs:
+  using: composite
+  steps:
+    - name: Extract target branch
+      id: extract
+      shell: bash
+      run: |
+        branch=$(echo "${INPUTS_TEXT}" | grep -ioP '\[(?:java|dotnet|python|ruby|php|golang|cpp|agent|nodejs)@[^]]+(?=\])' | tr -d '[:space:]' || true)
+
+        echo "target-branch=${branch#*@}" >> $GITHUB_OUTPUT
+
+      # the preferred approach to handling untrusted input is to set the value of the expression to an intermediate environment variable
+      env:
+          INPUTS_TEXT: ${{ inputs.text }}

.github/actions/lint_code/action.yaml

@@ -7,7 +7,7 @@ runs:
       uses: ./.github/actions/install_runner
     - run: source venv/bin/activate
       shell: bash
-    - name: Black, pylint and tailing whitespaces checks
+    - name: Black, pylint, tailing whitespaces, and yaml checks
       shell: bash
       run: ./format.sh --check
     - if: ${{ failure() }}
@@ -34,4 +34,3 @@ runs:
       run: |
         npm install
         npm run lint
-

.github/actions/pull_images/action.yml

@@ -3,12 +3,12 @@ description: "Pull docker images"
 inputs:
   library:
     description: "Which library will be tested (python, cpp, java, ...)"
-    required: true
+    default: ""
   weblog:
     description: "Which weblog will be tested"
-    required: true
+    default: ""
   scenarios:
-    description: "JSON array of scenarios that will be executed"
+    description: "JSON array, or comma separated list of scenarios that will be executed"
     required: true
   cleanup:
     description: "Whether to cleanup the disk to free up more space. Should be disabled when a larger machine can be used instead."
@@ -46,7 +46,7 @@ runs:
       shell: bash
       run: |
         source venv/bin/activate
-        python utils/scripts/get-image-list.py ${{ inputs.library }} ${{ inputs.weblog }} '${{ inputs.scenarios }}' > compose.yaml
+        python utils/scripts/get-image-list.py '${{ inputs.scenarios }}' -l=${{ inputs.library }} -w=${{ inputs.weblog }} > compose.yaml
       env:
         PYTHONPATH: "."
 

.github/workflows/ci.yml

@@ -10,6 +10,9 @@ on:
     - synchronize
     - labeled
     - unlabeled
+  push:
+    branches:
+    - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -50,34 +53,52 @@ jobs:
     uses: ./.github/workflows/compute-impacted-libraries.yml
 
   get_dev_artifacts:
+    # WARNING : never include something secret inside this job, as
+    # the result of it will be stored in a public artifact
+    # For instance, do not use GH_TOKEN here
     needs:
     - impacted_libraries
     strategy:
       matrix:
-        library: ${{ fromJson(needs.impacted_libraries.outputs.impacted_libraries) }}
+        library: ${{ fromJson(needs.impacted_libraries.outputs.libraries_with_dev) }}
       fail-fast: false
     runs-on: ubuntu-latest
+    outputs:
+      target-branch: ${{ steps.get-target-branch.outputs.target-branch }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Get target branch
+        uses: ./.github/actions/get_target_branch
+        id: get-target-branch
+        with:
+          text: ${{ github.event.pull_request.title }}
       - name: Get library artifact
         run: ./utils/scripts/load-binary.sh ${{ matrix.library }}
+        env:
+          TARGET_BRANCH: "${{ steps.get-target-branch.outputs.target-branch }}"
 
       - name: Get agent artifact
         run: ./utils/scripts/load-binary.sh agent
 
-      # ### appsec-event-rules is now a private repo. The GH_TOKEN provided can't read private repos.
-      # ### skipping this, waiting for a proper solution
-      # - name: Load WAF rules
-      #   if: matrix.version == 'dev'
-      #   run: ./utils/scripts/load-binary.sh waf_rule_set
-      #   env:
-      #     GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: binaries_dev_${{ matrix.library }}
           path: binaries/
+          include-hidden-files: ${{ matrix.library == 'python' }}
+
+  fail-if-target-branch:
+    name: Fail if target branch is specified
+    needs:
+      - get_dev_artifacts
+    if: needs.get_dev_artifacts.outputs.target-branch != ''
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if PR title contains a target branch
+        run: |
+          echo "This PR can't be merged, due to the title specifying a target branch"
+          exit 1
 
   system_tests:
     name: System Tests
@@ -89,10 +110,8 @@ jobs:
     - get_dev_artifacts
     strategy:
       matrix:
-        library: ${{ fromJson(needs.impacted_libraries.outputs.impacted_libraries) }}
-        version:
-        - prod
-        - dev
+        include: ${{ fromJson(needs.impacted_libraries.outputs.library_matrix) }}
+
       fail-fast: false
     uses: ./.github/workflows/system-tests.yml
     permissions:
@@ -109,8 +128,9 @@ jobs:
       build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
       build_lib_injection_app_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-lib-injection-app-images') }}
-      _experimental_parametric_job_count: ${{ matrix.version == 'dev' && 2 || 1 }}  # test both use cases
+      parametric_job_count: ${{ matrix.version == 'dev' && 2 || 1 }}  # test both use cases
       skip_empty_scenarios: true
+      desired_execution_time: 3600  # one hour
 
   system_tests_docker_mode:
     name: Ruby Docker Mode
@@ -119,7 +139,7 @@ jobs:
     - test_the_test
     - impacted_libraries
     - get_dev_artifacts  # non official set-up, this needs put this job in last
-    if: contains(needs.impacted_libraries.outputs.impacted_libraries, 'ruby')
+    if: contains(needs.impacted_libraries.outputs.library_matrix, 'ruby')
     uses: ./.github/workflows/run-docker-mode.yml
     permissions:
       packages: write
@@ -131,20 +151,24 @@ jobs:
     uses: ./.github/workflows/run-exotics.yml
     secrets: inherit
 
-  fancy-report:
+  all-jobs-are-green:  # if this must be renamed, you need to update utils/scripts/get-workflow-summary.py
+    name: All jobs are green
     runs-on: ubuntu-latest
     needs:
     - system_tests
-    if: always()
+    if: '!cancelled()'
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
         python-version: '3.11'
     - run: pip install requests
-    - run: python utils/scripts/get-workflow-summary.py ${{ github.run_id }} >> $GITHUB_STEP_SUMMARY
+    - run: python utils/scripts/get-workflow-summary.py DataDog/system-tests ${{ github.run_id }} -o $GITHUB_STEP_SUMMARY
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Fail if any job failed
+      if: needs.system_tests.result != 'success'
+      run: exit 1
 
   update-CI-visibility:
     name: Update CI Visibility Dashboard

.github/workflows/compute-impacted-libraries.yml

@@ -3,23 +3,27 @@ name: "Compute libraries impacted by the change"
 on:
   workflow_call:
     outputs:
-      impacted_libraries:
-        description: "JSON list of libraries impacted by the change"
-        value: ${{ jobs.main.outputs.impacted_libraries }}
+      libraries_with_dev:
+        description: "JSON list of libraries that require a dev run"
+        value: ${{ jobs.main.outputs.libraries_with_dev }}
+      library_matrix:
+        description: "List of couple library + dev/prod to run"
+        value: ${{ jobs.main.outputs.library_matrix }}
 
 jobs:
   main:
     runs-on: ubuntu-latest
     outputs:
-      impacted_libraries: ${{ steps.compute_impacted_libraries.outputs.result }}
+      library_matrix:  ${{ steps.compute.outputs.library_matrix }}
+      libraries_with_dev:  ${{ steps.compute.outputs.libraries_with_dev }}
     steps:
       - uses: actions/checkout@v4
       - run:  |
           git fetch origin ${{ github.event.pull_request.base.sha || github.sha }}
           git diff --name-only HEAD ${{ github.event.pull_request.base.sha || github.sha }} >> modified_files.txt
           cat modified_files.txt
       - name: Compute impacted libraries
-        id: compute_impacted_libraries
+        id: compute
         shell: python
         run: |
           import json
@@ -31,20 +35,24 @@ jobs:
           # temporary print to see what's hapenning on differents events
           print(json.dumps(github_context, indent=2))
 
-          libraries = "cpp|dotnet|golang|java|nodejs|php|python|ruby"
+          libraries = "cpp|dotnet|golang|java|nodejs|php|python|ruby|java_otel|python_otel|nodejs_otel"
           result = set()
 
+          # do not include otel in system-tests CI by default, as the staging backend is not stable enough
+          # all_libraries = {"cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby", "java_otel", "python_otel", "nodejs_otel"}
+          all_libraries = {"cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby"}
+
           if github_context["ref"] == "refs/heads/main":
               print("Merge commit to main => run all libraries")
-              result |= {"cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby"}
+              result |= all_libraries
 
           elif github_context["event_name"] == "schedule":
               print("Scheduled job => run all libraries")
-              result |= {"cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby"}
+              result |= all_libraries
 
           else:
               pr_title = github_context["event"]["pull_request"]["title"].lower()
-              match = re.search(rf"^\[({libraries})\]", pr_title)
+              match = re.search(rf"^\[({libraries})(?:@([^\]]+))?\]", pr_title)
               if match:
                   print(f"PR title matchs => run {match[1]}")
                   result.add(match[1])
@@ -67,15 +75,31 @@ jobs:
 
                       if not manifest_match and not weblog_match and not injection_match:
                           print(f"{file} may impact any library => run all of them")
-                          result |= {"cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby"}
+                          result |= all_libraries
                           break
 
+          populated_result = [
+            {
+              "library": library,
+              "version": "prod",
+            }
+            for library in sorted(result)
+          ] + [
+            {
+              "library": library,
+              "version": "dev",
+            }
+            for library in sorted(result)
+            if "otel" not in library
+          ]
+
           with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh:
-              print(f"result={json.dumps(sorted(result))}", file=fh)
+              print(f"library_matrix={json.dumps(populated_result)}", file=fh)
+              print(f"libraries_with_dev={json.dumps([item['library'] for item in populated_result if item['version'] == 'dev'])}", file=fh)
 
         env:
           GITHUB_CONTEXT: ${{ toJSON(github) }}
 
       - name: Print results
         run: |
-          echo 'Impacted libraries: -> ${{ steps.compute_impacted_libraries.outputs.result }}'
+          echo 'Impacted libraries: -> ${{ steps.compute.outputs.library_matrix }}'