[main] Source code updates from dotnet/dotnet

eng/common/core-templates/job/job.yml

@@ -31,6 +31,7 @@ parameters:
   testRunTitle: ''
   testResultsFormat: ''
   name: ''
+  componentGovernanceSteps: []
   preSteps: []
   artifactPublishSteps: []
   runAsPublic: false
@@ -149,6 +150,9 @@ jobs:
   - ${{ each step in parameters.steps }}:
     - ${{ step }}
 
+  - ${{ each step in parameters.componentGovernanceSteps }}:
+    - ${{ step }}
+
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
       parameters:

eng/common/core-templates/job/publish-build-assets.yml

@@ -122,8 +122,9 @@ jobs:
 
     # Populate internal runtime variables.
     - template: /eng/common/templates/steps/enable-internal-sources.yml
-      parameters:
-        legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
+      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+        parameters:
+            legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
 
     - template: /eng/common/templates/steps/enable-internal-runtimes.yml
 
@@ -172,7 +173,7 @@ jobs:
             artifactName: AssetManifests
             displayName: 'Publish Merged Manifest'
             retryCountOnTaskFailure: 10 # for any logs being locked
-            isProduction: false
+            sbomEnabled: false # we don't need SBOM for logs
 
     - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
       parameters:

eng/common/core-templates/job/source-build.yml

@@ -60,7 +60,7 @@ jobs:
       pool:
         ${{ if eq(variables['System.TeamProject'], 'public') }}:
           name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
-          demands: ImageOverride -equals build.ubuntu.2004.amd64
+          demands: ImageOverride -equals build.ubuntu.2204.amd64
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:
           name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
           image: 1es-mariner-2

eng/common/core-templates/post-build/post-build.yml

@@ -305,8 +305,9 @@ stages:
           PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
 
-      - task: NuGetAuthenticate@1 # Populate internal runtime variables.
+      - task: NuGetAuthenticate@1
 
+      # Populate internal runtime variables.
       - template: /eng/common/templates/steps/enable-internal-sources.yml
         parameters:
           legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)

eng/common/core-templates/steps/component-governance.yml

@@ -0,0 +1,16 @@
+parameters:
+  disableComponentGovernance: false
+  componentGovernanceIgnoreDirectories: ''
+  is1ESPipeline: false
+  displayName: 'Component Detection'
+
+steps:
+- ${{ if eq(parameters.disableComponentGovernance, 'true') }}:
+  - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true"
+    displayName: Set skipComponentGovernanceDetection variable
+- ${{ if ne(parameters.disableComponentGovernance, 'true') }}:
+  - task: ComponentGovernanceComponentDetection@0
+    continueOnError: true
+    displayName: ${{ parameters.displayName }}
+    inputs:
+      ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}

eng/common/core-templates/steps/generate-sbom.yml

@@ -1,14 +1,54 @@
+# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
+# PackageName - The name of the package this SBOM represents.
+# PackageVersion - The version of the package this SBOM represents. 
+# ManifestDirPath - The path of the directory where the generated manifest files will be placed
+# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
+
 parameters:
-  PackageVersion: unused
-  BuildDropPath: unused
-  PackageName: unused
-  ManifestDirPath: unused
-  IgnoreDirectories: unused
-  sbomContinueOnError: unused
-  is1ESPipeline: unused
-  publishArtifacts: unused
+  PackageVersion: 11.0.0
+  BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts'
+  PackageName: '.NET'
+  ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
+  IgnoreDirectories: ''
+  sbomContinueOnError: true
+  is1ESPipeline: false
+  # disable publishArtifacts if some other step is publishing the artifacts (like job.yml).
+  publishArtifacts: true
 
 steps:
+- task: PowerShell@2 
+  displayName: Prep for SBOM generation in (Non-linux)
+  condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
+  inputs: 
+    filePath: ./eng/common/generate-sbom-prep.ps1
+    arguments: ${{parameters.manifestDirPath}}
+
+# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
 - script: |
-    echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include."
-  displayName: Issue generate-sbom.yml deprecation warning
+    chmod +x ./eng/common/generate-sbom-prep.sh
+    ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
+  displayName: Prep for SBOM generation in (Linux)
+  condition: eq(variables['Agent.Os'], 'Linux')
+  continueOnError: ${{ parameters.sbomContinueOnError }}
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+  displayName: 'Generate SBOM manifest'
+  continueOnError: ${{ parameters.sbomContinueOnError }}
+  inputs:
+      PackageName: ${{ parameters.packageName }}
+      BuildDropPath: ${{ parameters.buildDropPath }}
+      PackageVersion: ${{ parameters.packageVersion }}
+      ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME)
+      ${{ if ne(parameters.IgnoreDirectories, '') }}:
+        AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
+
+- ${{ if eq(parameters.publishArtifacts, 'true')}}:
+  - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      args:
+        displayName: Publish SBOM manifest
+        continueOnError: ${{parameters.sbomContinueOnError}}
+        targetPath: '${{ parameters.manifestDirPath }}'
+        artifactName: $(ARTIFACT_NAME)
+

eng/common/core-templates/steps/source-build.yml

@@ -62,4 +62,4 @@ steps:
       artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
       continueOnError: true
       condition: succeededOrFailed()
-      isProduction: false
+      sbomEnabled: false  # we don't need SBOM for logs

eng/common/generate-sbom-prep.ps1

@@ -0,0 +1,29 @@
+Param(
+    [Parameter(Mandatory=$true)][string] $ManifestDirPath    # Manifest directory where sbom will be placed
+)
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
+Write-Host "Creating dir $ManifestDirPath"
+
+# create directory for sbom manifest to be placed
+if (!(Test-Path -path $SbomGenerationDir))
+{
+  New-Item -ItemType Directory -path $SbomGenerationDir
+  Write-Host "Successfully created directory $SbomGenerationDir"
+}
+else{
+  Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
+}
+
+Write-Host "Updating artifact name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"

eng/common/generate-sbom-prep.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+. $scriptroot/pipeline-logging-functions.sh
+
+
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
+manifest_dir=$1
+
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+sbom_generation_dir="$manifest_dir/$safe_artifact_name"
+
+if [ ! -d "$sbom_generation_dir" ] ; then
+  mkdir -p "$sbom_generation_dir"
+  echo "Sbom directory created." $sbom_generation_dir
+else
+  Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
+fi
+
+echo "Artifact name before : "$artifact_name
+echo "Artifact name after : "$safe_artifact_name
+export ARTIFACT_NAME=$safe_artifact_name
+echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
+
+exit 0

eng/common/template-guidance.md

@@ -82,6 +82,7 @@ eng\common\
             publish-build-artifacts.yml      (logic)
             publish-pipeline-artifacts.yml   (logic)
             component-governance.yml         (shim)
+            generate-sbom.yml                (shim)
             publish-logs.yml                 (shim)
             retain-build.yml                 (shim)
             send-to-helix.yml                (shim)
@@ -106,6 +107,7 @@ eng\common\
             setup-maestro-vars.yml           (logic)
         steps\
             component-governance.yml         (logic)
+            generate-sbom.yml                (logic)
             publish-build-artifacts.yml      (redirect)
             publish-logs.yml                 (logic)
             publish-pipeline-artifacts.yml   (redirect)

eng/common/templates-official/job/job.yml

@@ -1,15 +1,24 @@
 parameters:
+# Sbom related params
+  enableSbom: true
   runAsPublic: false
-# Sbom related params, unused now and can eventually be removed
-  enableSbom: unused
-  PackageVersion: unused
-  BuildDropPath: unused
+  PackageVersion: 9.0.0
+  BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts'
 
 jobs:
 - template: /eng/common/core-templates/job/job.yml
   parameters:
     is1ESPipeline: true
 
+    componentGovernanceSteps:
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+      - template: /eng/common/templates/steps/generate-sbom.yml
+        parameters:
+          PackageVersion: ${{ parameters.packageVersion }}
+          BuildDropPath: ${{ parameters.buildDropPath }}
+          ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
+          publishArtifacts: false
+
     # publish artifacts
     # for 1ES managed templates, use the templateContext.output to handle multiple outputs.
     templateContext:
@@ -32,7 +41,7 @@ jobs:
             continueOnError: true
             condition: always()
             retryCountOnTaskFailure: 10 # for any logs being locked
-            isProduction: false
+            sbomEnabled: false  # we don't need SBOM for logs
 
       - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}:
         - output: buildArtifacts
@@ -42,15 +51,22 @@ jobs:
           ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
           continueOnError: true
           condition: always()
-          #isProduction: false
+          sbomEnabled: false  # we don't need SBOM for logs
 
       - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
         - output: pipelineArtifact
           targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/eng/common/BuildConfiguration'
           artifactName: 'BuildConfiguration'
           displayName: 'Publish build retry configuration'
           continueOnError: true
-          isProduction: false
+          sbomEnabled: false  # we don't need SBOM for BuildConfiguration
+
+      - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+        - output: pipelineArtifact
+          displayName: Publish SBOM manifest
+          continueOnError: true
+          targetPath: $(Build.ArtifactStagingDirectory)/sbom
+          artifactName: $(ARTIFACT_NAME)
 
       # add any outputs provided via root yaml
       - ${{ if ne(parameters.templateContext.outputs, '') }}:

eng/common/templates-official/steps/component-governance.yml

@@ -0,0 +1,7 @@
+steps:
+- template: /eng/common/core-templates/steps/component-governance.yml
+  parameters:
+    is1ESPipeline: true
+
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/steps/publish-pipeline-artifacts.yml

@@ -26,5 +26,3 @@ steps:
       properties: ${{ parameters.args.properties }}
     ${{ if parameters.args.sbomEnabled }}:
       sbomEnabled: ${{ parameters.args.sbomEnabled }}
-    ${{ if parameters.args.isProduction }}:
-      isProduction: ${{ parameters.args.isProduction }}

eng/common/templates/job/job.yml

@@ -1,12 +1,12 @@
 parameters: 
   enablePublishBuildArtifacts: false
+  disableComponentGovernance: ''
+  componentGovernanceIgnoreDirectories: ''
+# Sbom related params
+  enableSbom: true
   runAsPublic: false
-# CG related params, unused now and can eventually be removed
-  disableComponentGovernance: unused
-# Sbom related params, unused now and can eventually be removed
-  enableSbom: unused
-  PackageVersion: unused
-  BuildDropPath: unused
+  PackageVersion: 9.0.0
+  BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts'
 
 jobs:
 - template: /eng/common/core-templates/job/job.yml
@@ -21,10 +21,17 @@ jobs:
     - ${{ each step in parameters.steps }}:
       - ${{ step }}
 
-    # we don't run CG in public
-    - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-      - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true"
-        displayName: Set skipComponentGovernanceDetection variable
+    componentGovernanceSteps:
+    - template: /eng/common/templates/steps/component-governance.yml
+      parameters:
+        ${{ if eq(parameters.disableComponentGovernance, '') }}:
+          ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+            disableComponentGovernance: false
+          ${{ else }}:
+            disableComponentGovernance: true
+        ${{ else }}:
+          disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
+        componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
 
     artifactPublishSteps:
     - ${{ if ne(parameters.artifacts.publish, '') }}:
@@ -51,7 +58,7 @@ jobs:
               continueOnError: true
               condition: always()
               retryCountOnTaskFailure: 10 # for any logs being locked
-              isProduction: false
+              sbomEnabled: false  # we don't need SBOM for logs
 
     - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
       - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
@@ -74,4 +81,4 @@ jobs:
             artifactName: 'BuildConfiguration'
             displayName: 'Publish build retry configuration'
             continueOnError: true
-            isProduction: false
+            sbomEnabled: false  # we don't need SBOM for BuildConfiguration

eng/common/templates/steps/component-governance.yml

@@ -0,0 +1,7 @@
+steps:
+- template: /eng/common/core-templates/steps/component-governance.yml
+  parameters:
+    is1ESPipeline: false
+
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

global.json

@@ -32,9 +32,9 @@
     "jdk": "latest"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.25569.110",
-    "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.25569.110",
-    "Microsoft.DotNet.SharedFramework.Sdk": "11.0.0-beta.25569.110",
+    "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.25570.101",
+    "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.25570.101",
+    "Microsoft.DotNet.SharedFramework.Sdk": "11.0.0-beta.25570.101",
     "Microsoft.Build.NoTargets": "3.7.0",
     "Microsoft.Build.Traversal": "3.4.0",
     "Microsoft.WixToolset.Sdk": "5.0.2-dotnet.2811440"