walk: generated string walker for references

walk.Strings visits the strings of the configuration with plain field access
(generated, one function per struct type that contains strings), filtered
inline: WithReferences passes the strings that contain "${". The callback
can return a replacement, which is stored in the field. RewriteWorkspacePrefix,
TFOnlyReferences and reverse interpolation use it instead of walking every
value of the configuration; pure references in non-string fields come from
Root.References.

1000 jobs (main / #6954 / this): RewriteWorkspacePrefix 114 / 19 / 1.8 ms,
TFOnlyReferences 91 / 19 / 1.9 ms.

Co-authored-by: Isaac <no-reply@databricks.com>

Author: denik

.agents/rules/auto-generated-files.md

@@ -15,6 +15,7 @@ globs:
   - "internal/mocks/**/*.go"
   - "bundle/direct/dresources/configs/*.generated.yml"
   - "bundle/config/validate/*_walk.go"
+  - "bundle/config/walk/*_walk.go"
   - "bundle/schema/jsonschema.json"
   - "python/databricks/bundles/version.py"
   - "python/databricks/bundles/*/__init__.py"
@@ -34,6 +35,7 @@ paths:
   - "internal/mocks/**/*.go"
   - "bundle/direct/dresources/configs/*.generated.yml"
   - "bundle/config/validate/*_walk.go"
+  - "bundle/config/walk/*_walk.go"
   - "bundle/schema/jsonschema.json"
   - "python/databricks/bundles/version.py"
   - "python/databricks/bundles/*/__init__.py"

.agents/skills/bump-sdk/SKILL.md

@@ -9,7 +9,7 @@ allowed-tools: Read, Edit, Write, Bash, Glob, Grep, WebFetch, AskUserQuestion
 
 The SDK version lives in `go.mod` (`github.com/databricks/databricks-sdk-go`) and the pinned spec SHA lives in `.codegen/_openapi_sha`.
 These two move as a pair; everything else in this skill is regenerated from them or is fallout you fix by hand.
-Do not hand-edit generated files (`.codegen/cli.json`, `cmd/workspace/*`, `cmd/account/*`, `bundle/schema/jsonschema.json`, `bundle/config/validate/*_walk.go`, `bundle/direct/dresources/configs/*.generated.yml`, `python/databricks/bundles/**`); regenerate them.
+Do not hand-edit generated files (`.codegen/cli.json`, `cmd/workspace/*`, `cmd/account/*`, `bundle/schema/jsonschema.json`, `bundle/config/validate/*_walk.go`, `bundle/config/walk/*_walk.go`, `bundle/direct/dresources/configs/*.generated.yml`, `python/databricks/bundles/**`); regenerate them.
 
 The Python tasks (`pydabs-*`, and the `pydabs-codegen` step inside `generate-check`) all run through `uv`. If one fails because `uv` is missing or because the host's `python3` is too old (e.g. 3.9), install `uv` (`curl -LsSf https://astral.sh/uv/install.sh | sh`) rather than touching the system Python: `uv run` provisions the interpreter each package pins (`>=3.10`, and `==3.13.*` under `python/codegen/`) and downloads it if needed. Do not chase the system Python version.
 The acceptance goldens are the exception: they need CI's `uv` index, see step 4.

Taskfile.yml

@@ -1002,6 +1002,7 @@ tasks:
       - "{{.EMBED_SOURCES}}"
     generates:
       - bundle/config/validate/*_walk.go
+      - bundle/config/walk/*_walk.go
     cmds:
       - "go run ./bundle/internal/validation/."
 

bundle/config/mutator/rewrite_workspace_prefix.go

@@ -6,9 +6,9 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/walk"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/structs/structpath"
-	"github.com/databricks/cli/libs/structs/structvar"
 )
 
 type rewriteWorkspacePrefix struct{}
@@ -47,12 +47,7 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
 
 	// Walk through the bundle configuration, check all the string leafs and
 	// see if any of the prefixes are used in the remote path.
-	err := structvar.Walk(b.Config.View(), func(p *structpath.PathNode, v structvar.View) error {
-		vv, ok := v.AsString()
-		if !ok {
-			return nil
-		}
-
+	walk.Strings(&b.Config, walk.WithReferences, func(n *walk.Node, vv string) (string, bool) {
 		newPath := vv
 		for _, rewrite := range paths {
 			if !strings.Contains(newPath, rewrite.pattern) {
@@ -64,21 +59,14 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
 				Severity:  diag.Warning,
 				Summary:   fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath),
 				Detail:    "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths",
-				Locations: v.Locations(),
-				Paths:     []*structpath.PathNode{p},
+				Locations: n.Locations().Get(),
+				Paths:     []*structpath.PathNode{n.Path()},
 			})
 		}
 
-		if newPath == vv {
-			return nil
-		}
-
 		// Remove the workspace prefix from the string.
-		return b.Config.Set(p, newPath)
+		return newPath, newPath != vv
 	})
-	if err != nil {
-		return diag.FromErr(err)
-	}
 
 	return diags
 }

bundle/config/validate/tf_only_references.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/walk"
 	"github.com/databricks/cli/bundle/terraform_dabs_map"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/structs/structpath"
@@ -28,26 +29,15 @@ func (m *tfOnlyReferences) Name() string {
 func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	var diags diag.Diagnostics
 
-	// Walk the entire config looking for ${resources.*} references.
-	_ = structvar.Walk(b.Config.View(), func(_ *structpath.PathNode, v structvar.View) error {
-		s, ok := v.AsString()
-		if !ok {
-			return nil
-		}
-		ref, ok := structvar.NewRef(s)
-		if !ok {
-			return nil
-		}
-		for _, r := range ref.References() {
-			if !strings.HasPrefix(r, "resources.") {
-				continue
-			}
-			if d := checkTFOnlyReference(r, v.Location()); d != nil {
-				diags = append(diags, *d)
-			}
-		}
-		return nil
+	// Look for ${resources.*} references in strings and in fields that hold a pure
+	// reference.
+	walk.Strings(&b.Config, walk.WithReferences, func(n *walk.Node, s string) (string, bool) {
+		diags = append(diags, checkTFOnlyReferences(s, n.Location())...)
+		return "", false
 	})
+	for p, ref := range b.Config.References() {
+		diags = append(diags, checkTFOnlyReferences(ref, b.Config.ExactLocation(p))...)
+	}
 
 	if len(diags) > 0 {
 		b.Metrics.AddBoolValue("has_tf_only_references", true)
@@ -56,6 +46,24 @@ func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagn
 	return diags
 }
 
+// checkTFOnlyReferences checks the ${resources.*} references in s.
+func checkTFOnlyReferences(s string, loc diag.Location) diag.Diagnostics {
+	ref, ok := structvar.NewRef(s)
+	if !ok {
+		return nil
+	}
+	var diags diag.Diagnostics
+	for _, r := range ref.References() {
+		if !strings.HasPrefix(r, "resources.") {
+			continue
+		}
+		if d := checkTFOnlyReference(r, loc); d != nil {
+			diags = append(diags, *d)
+		}
+	}
+	return diags
+}
+
 // checkTFOnlyReference checks a single reference string like
 // "resources.jobs.src.always_running" and returns a diagnostic when it refers
 // to a TF-only field, or nil otherwise.

bundle/config/walk/.gitattributes

@@ -0,0 +1 @@
+*_walk.go linguist-generated=true

bundle/config/walk/node.go

@@ -4,6 +4,7 @@ package walk
 
 import (
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/structs/structvar"
 )
@@ -52,6 +53,15 @@ func (n *Node) Locations() *structvar.Locations {
 	return n.locs
 }
 
+// Location returns the first location of the value at n.
+func (n *Node) Location() diag.Location {
+	locs := n.locs.Get()
+	if len(locs) == 0 {
+		return diag.Location{}
+	}
+	return locs[0]
+}
+
 // Written reports whether the key k of n is written in the source.
 func (n *Node) Written(k string) bool {
 	return n.locs.Key(k) != nil

bundle/config/walk/strings.go

@@ -0,0 +1,76 @@
+package walk
+
+import (
+	"maps"
+	"slices"
+	"strings"
+
+	"github.com/databricks/cli/bundle/config"
+)
+
+// Mode selects the strings [Strings] visits.
+type Mode int
+
+const (
+	// WithReferences visits the strings that contain a variable reference ("${").
+	WithReferences Mode = iota
+)
+
+// StringFunc is called for a visited string s at n. It returns a replacement and
+// whether to store it.
+type StringFunc func(n *Node, s string) (string, bool)
+
+// Strings calls fn for the strings of the configuration selected by mode, with plain
+// field access (strings_walk.go is generated by bundle/internal/validation). Pure
+// references in fields that cannot hold a string are not strings: see
+// [config.Root.References].
+//
+// TODO: processors that run back to back (e.g. reference collection and the
+// workspace-prefix rewrite) could share one walk: visit every string with its kind
+// (empty, plain, containing "${", pure reference) and dispatch to each processor,
+// paying for the traversal once. Phase ordering limits which ones can be combined.
+func Strings(root *config.Root, mode Mode, fn StringFunc) {
+	c := &stringsWalk{mode: mode, fn: fn}
+	n := Root(root)
+	stringsConfigRoot(c, &n, root)
+}
+
+type stringsWalk struct {
+	mode Mode
+	fn   StringFunc
+}
+
+func (c *stringsWalk) match(s string) bool {
+	return strings.Contains(s, "${")
+}
+
+// any visits the strings of a dynamic value (e.g. a variable value): the maps, slices
+// and strings that decoding produces.
+func (c *stringsWalk) any(n *Node, v *any) {
+	switch x := (*v).(type) {
+	case string:
+		if c.match(x) {
+			if r, ok := c.fn(n, x); ok {
+				*v = r
+			}
+		}
+	case map[string]any:
+		for _, k := range sortedKeys(x) {
+			e := x[k]
+			ch := n.Child(k)
+			c.any(&ch, &e)
+			x[k] = e
+		}
+	case []any:
+		for i := range x {
+			ch := n.Elem(i)
+			c.any(&ch, &x[i])
+		}
+	default:
+	}
+}
+
+// sortedKeys returns the keys of m in the order maps are visited.
+func sortedKeys[K ~string, V any](m map[K]V) []K {
+	return slices.Sorted(maps.Keys(m))
+}