Do not write job defaults into a reference to a whole block (#7225)

## Changes
`bundle.SetDefault` skips a matched value that is a pure `${...}`
reference instead of indexing into it. Applies to every default set in
`resourcemutator` (e.g. `schedule`/`trigger`/`continuous.pause_status`).

## Why
`schedule: ${resources.jobs.base.schedule}` made `bundle validate` fail
with `expected a map to index "pause_status", found string`. The
referenced schedule gets the default itself, so the resolved value has
it (see the plan output in the test).

Port note: the fix lives in `bundle/set_default.go` and is dyn-specific
(a reference is a string in the dyn tree); on typed structs the
equivalent is to skip a field held as a reference.

## Tests
- Unit test in `bundle/set_default_test.go`.
-
`acceptance/bundle/validate/references/job_{schedule,trigger}_whole_reference`:
Badness removed; validate and plan succeed. The schedule test keeps a
Badness for the spurious required-field warnings, fixed in the
follow-up.

This pull request and its description were written by Isaac.

---------

Co-authored-by: Isaac <no-reply@databricks.com>

Author: denik

.nextchanges/bundles/defaults-whole-reference.md

@@ -0,0 +1 @@
+* Fixed `bundle validate` failing with "expected a map to index" when a job `schedule`, `trigger` or `continuous` is a reference to a whole block, such as `${resources.jobs.base.schedule}`. ([#7225](https://github.com/databricks/cli/pull/7225))

acceptance/bundle/validate/references/job_schedule_whole_reference/output.txt

@@ -1,11 +1,32 @@
 
->>> musterr [CLI] bundle validate
-Error: expected a map to index "pause_status", found string
+>>> [CLI] bundle validate
+Warning: required field "quartz_cron_expression" is not set
+  at resources.jobs.viaref.schedule
+  in databricks.yml:9:17
+
+Warning: required field "timezone_id" is not set
+  at resources.jobs.viaref.schedule
+  in databricks.yml:9:17
 
 Name: test-bundle
 Target: default
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
-Found 1 error
+Found 2 warnings
+
+>>> [CLI] bundle plan -o json
+Warning: required field "quartz_cron_expression" is not set
+  at resources.jobs.viaref.schedule
+  in databricks.yml:9:17
+
+Warning: required field "timezone_id" is not set
+  at resources.jobs.viaref.schedule
+  in databricks.yml:9:17
+
+{
+  "pause_status": "UNPAUSED",
+  "quartz_cron_expression": "0 0 * * * ?",
+  "timezone_id": "UTC"
+}

acceptance/bundle/validate/references/job_schedule_whole_reference/script

@@ -1,3 +1,4 @@
 # The schedule of viaref is a reference to the whole schedule of another job.
-# The defaults set schedule.pause_status (and trigger, continuous) of every job by path, which cannot be done in a reference.
-trace musterr $CLI bundle validate
+# The defaults (e.g. pause_status) are not written into the reference; the referenced schedule gets them.
+trace $CLI bundle validate
+trace $CLI bundle plan -o json | jq '.plan."resources.jobs.viaref".new_state.value.schedule'

acceptance/bundle/validate/references/job_schedule_whole_reference/test.toml

@@ -1 +1 @@
-Badness = "The default of schedule.pause_status is written by path, which fails with 'expected a map to index pause_status, found string' when the schedule is a reference to a whole schedule of another job; validate should accept it."
+Badness = "The required-field check looks inside the reference: it warns that quartz_cron_expression and timezone_id are not set at resources.jobs.viaref.schedule, although the referenced schedule sets both."

acceptance/bundle/validate/references/job_trigger_whole_reference/output.txt

@@ -1,11 +1,17 @@
 
->>> musterr [CLI] bundle validate
-Error: expected a map to index "pause_status", found string
-
+>>> [CLI] bundle validate
 Name: test-bundle
 Target: default
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
-Found 1 error
+Validation OK!
+
+>>> [CLI] bundle plan -o json
+{
+  "file_arrival": {
+    "url": "s3://bucket/path"
+  },
+  "pause_status": "UNPAUSED"
+}

acceptance/bundle/validate/references/job_trigger_whole_reference/script

@@ -1,3 +1,4 @@
 # The trigger of viaref is a reference to the whole trigger of another job.
-# The defaults set trigger.pause_status (and trigger, continuous) of every job by path, which cannot be done in a reference.
-trace musterr $CLI bundle validate
+# The defaults (e.g. pause_status) are not written into the reference; the referenced trigger gets them.
+trace $CLI bundle validate
+trace $CLI bundle plan -o json | jq '.plan."resources.jobs.viaref".new_state.value.trigger'

acceptance/bundle/validate/references/job_trigger_whole_reference/test.toml

@@ -6,6 +6,7 @@ import (
 
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/logdiag"
 )
 
@@ -30,6 +31,11 @@ func (m *setDefault) Name() string {
 func (m *setDefault) Apply(ctx context.Context, b *Bundle) diag.Diagnostics {
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		return dyn.MapByPattern(v, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			// A default cannot be written into a pure reference to a whole block
+			// (e.g. "${resources.jobs.base.schedule}"), so leave it to the referenced value.
+			if s, ok := v.AsString(); ok && dynvar.IsPureVariableReference(s) {
+				return v, nil
+			}
 			_, err := dyn.GetByPath(v, m.key)
 			switch {
 			case dyn.IsNoSuchKeyError(err):

bundle/set_default.go

@@ -0,0 +1,39 @@
+package bundle
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/logdiag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetDefaultSkipsPureReference(t *testing.T) {
+	root, diags := config.LoadFromBytes("databricks.yml", []byte(`
+resources:
+  jobs:
+    base:
+      schedule:
+        quartz_cron_expression: "0 0 * * * ?"
+    viaref:
+      schedule: ${resources.jobs.base.schedule}
+`))
+	require.Empty(t, diags)
+	b := &Bundle{Config: *root}
+
+	ctx := logdiag.InitContext(t.Context())
+	logdiag.SetCollect(ctx, true)
+	SetDefault(ctx, b, "resources.jobs.*.schedule.pause_status", "UNPAUSED")
+	require.Empty(t, logdiag.FlushCollected(ctx))
+
+	v := b.Config.Value()
+	base, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.base.schedule.pause_status"))
+	require.NoError(t, err)
+	assert.Equal(t, "UNPAUSED", base.MustString())
+
+	viaref, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.viaref.schedule"))
+	require.NoError(t, err)
+	assert.Equal(t, "${resources.jobs.base.schedule}", viaref.MustString())
+}