[air] Remove yaml dependencies requirement when environment.version is specified (#6718)

## Changes
- Allow `environment.version` without requiring
`environment.dependencies`.
- Cover omitted, empty, and non-empty dependency lists and submission
payload behavior.
- Update the environment version help text.

## Why
`environment.version` is submitted independently from dependencies.
Requiring `dependencies: []` adds no-op YAML because omitted and empty
dependencies produce the same payload.

## Tests
- Focused AIR config and payload unit tests.
- AIR config-help and submit acceptance tests.
- Live AIR version 5 workload without dependencies completed
successfully.

_This PR was written with Codex._

Co-authored-by: Caroline Chen <324939130+caroline-db@users.noreply.github.com>

Author: caroline-db

acceptance/experimental/air/config-help/output.txt

@@ -99,6 +99,13 @@ config.environment.dependencies
   Type:     list of strings
   Required: no
 
+>>> [CLI] experimental air run -h config.environment.version
+config.environment.version
+  Client image version to pin.
+
+  Type:     string or int
+  Required: no
+
 === leaf field
 >>> [CLI] experimental air run -h config.compute.accelerator_type
 config.compute.accelerator_type

acceptance/experimental/air/config-help/script

@@ -17,6 +17,7 @@ trace $CLI experimental air run -h config.mlflow_artifact_location
 trace $CLI experimental air run -h config.compute.pool_id
 trace $CLI experimental air run -h config.compute.priority_class
 trace $CLI experimental air run -h config.environment.dependencies
+trace $CLI experimental air run -h config.environment.version
 
 title "leaf field"
 trace $CLI experimental air run -h config.compute.accelerator_type

acceptance/experimental/air/run-submit-databricks-ai/output.txt

@@ -1,5 +1,5 @@
 
-=== submit with a Databricks AI environment
+=== submit with a Databricks AI environment and no dependencies
 >>> [CLI] experimental air run -f run.yaml
 Submitting experiment: databricks-ai-v5-smoke
 Submitted workload with Job Run ID: 555
@@ -9,7 +9,7 @@ Tip: use --watch when submitting a run to stream logs to your terminal.
 Stream logs after submission using:
   databricks experimental air logs 555
 
-=== submit accepts a Databricks AI environment version
+=== submit includes the Databricks AI environment version and omits dependencies
 >>> print_requests.py //api/2.2/jobs/runs/submit
 {
   "method": "POST",
@@ -19,10 +19,7 @@ Stream logs after submission using:
       {
         "environment_key": "default",
         "spec": {
-          "base_environment": "workspace-base-environments/databricks_ai_v5",
-          "dependencies": [
-            "accelerate"
-          ]
+          "base_environment": "workspace-base-environments/databricks_ai_v5"
         }
       }
     ],

acceptance/experimental/air/run-submit-databricks-ai/run.yaml

@@ -5,5 +5,3 @@ compute:
   num_accelerators: 1
 environment:
   version: databricks_ai_v5
-  dependencies:
-    - accelerate

acceptance/experimental/air/run-submit-databricks-ai/script

@@ -1,5 +1,5 @@
-title "submit with a Databricks AI environment"
+title "submit with a Databricks AI environment and no dependencies"
 trace $CLI experimental air run -f run.yaml
 
-title "submit accepts a Databricks AI environment version"
+title "submit includes the Databricks AI environment version and omits dependencies"
 trace print_requests.py //api/2.2/jobs/runs/submit

experimental/air/cmd/runconfig.go

@@ -253,7 +253,7 @@ func validateSecretRefs(secrets map[string]string) error {
 // settings.
 type environmentConfig struct {
 	Dependencies      dependencies `yaml:"dependencies" help:"Inline list of packages to install. Not allowed alongside unity_catalog_image."`
-	Version           stringOrInt  `yaml:"version" help:"Client image version to pin. Only valid alongside inline dependencies."`
+	Version           stringOrInt  `yaml:"version" help:"Client image version to pin."`
 	UnityCatalogImage string       `yaml:"unity_catalog_image" help:"Unity Catalog custom image to run the workload on, as <catalog>.<schema>.<image>:<tag>. Not allowed alongside dependencies or version."`
 }
 
@@ -279,11 +279,6 @@ func (e *environmentConfig) validate() error {
 		return nil
 	}
 
-	// version pins the client image version, which is only meaningful alongside an
-	// inline dependency set.
-	if e.Version.set && !e.Dependencies.set {
-		return errors.New("'environment.version' requires inline 'dependencies' (a list of packages)")
-	}
 	if e.Version.set {
 		version, err := validateRuntimeVersion(e.Version.raw, "environment.version")
 		if err != nil {

experimental/air/cmd/runconfig_test.go

@@ -332,7 +332,15 @@ func TestEnvironmentConfigValidate(t *testing.T) {
 		{
 			"version without deps",
 			environmentConfig{Version: stringOrInt{set: true, raw: "5"}},
-			"requires inline 'dependencies'",
+			"",
+		},
+		{
+			"version with empty deps",
+			environmentConfig{
+				Version:      stringOrInt{set: true, raw: "5"},
+				Dependencies: dependencies{set: true, list: []string{}},
+			},
+			"",
 		},
 		{
 			"version with inline deps ok",

experimental/air/cmd/runsubmit_test.go

@@ -65,6 +65,9 @@ func TestBuildSubmitPayload(t *testing.T) {
 	assert.Equal(t, aiRuntimeEnvironmentKey, p.Environments[0].EnvironmentKey)
 	require.NotNil(t, p.Environments[0].Spec)
 	assert.Equal(t, "5", p.Environments[0].Spec.EnvironmentVersion)
+	environmentJSON, err := json.Marshal(p.Environments[0].Spec)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"environment_version":"5"}`, string(environmentJSON))
 
 	require.Len(t, p.Tasks, 1)
 	task := p.Tasks[0]