Wait for grants to provision as part of depends on

.nextchanges/bundles/grants-before-dependents.md

@@ -0,0 +1 @@
+* Resources that depend on another resource are now deployed after that resource's `grants`, through an implicit `lifecycle.depends_on` entry, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. ([#7142](https://github.com/databricks/cli/pull/7142))

acceptance/bundle/migrate/command/grants/out.new_state.json

@@ -16,6 +16,10 @@
     {
      "node": "resources.schemas.my_schema",
      "label": "${resources.schemas.my_schema.name}"
+    },
+    {
+     "node": "resources.schemas.my_schema.grants",
+     "label": "depends_on"
     }
    ]
   },
@@ -81,6 +85,10 @@
     {
      "node": "resources.schemas.my_schema",
      "label": "${resources.schemas.my_schema.name}"
+    },
+    {
+     "node": "resources.schemas.my_schema.grants",
+     "label": "depends_on"
     }
    ]
   },

acceptance/bundle/resources/grants/schemas/pipeline_run_as/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: schema-grants-pipeline-$UNIQUE_NAME
+
+resources:
+  catalogs:
+    pipeline_catalog:
+      name: catalog_grants_pipeline_$UNIQUE_NAME
+      grants:
+        - principal: $RUN_AS_SP
+          privileges:
+            - USE_CATALOG
+
+  schemas:
+    src:
+      catalog_name: ${resources.catalogs.pipeline_catalog.name}
+      name: src
+      grants:
+        - principal: $RUN_AS_SP
+          privileges:
+            - USE_SCHEMA
+            - CREATE_TABLE
+            - CREATE_MATERIALIZED_VIEW
+
+  pipelines:
+    load:
+      name: schema-grants-pipeline-$UNIQUE_NAME
+      catalog: ${resources.schemas.src.catalog_name}
+      schema: ${resources.schemas.src.name}
+      serverless: true
+      libraries:
+        - file:
+            path: ./transformation.py
+      run_as:
+        service_principal_name: $RUN_AS_SP

acceptance/bundle/resources/grants/schemas/pipeline_run_as/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt

@@ -0,0 +1,101 @@
+
+=== The grants of a referenced catalog or schema become an implicit lifecycle.depends_on
+>>> [CLI] bundle validate -o json
+{
+  "catalogs": {
+    "pipeline_catalog": null
+  },
+  "pipelines": {
+    "load": [
+      {
+        "resource": "schemas.src.grants"
+      }
+    ]
+  },
+  "schemas": {
+    "src": [
+      {
+        "resource": "catalogs.pipeline_catalog.grants"
+      }
+    ]
+  }
+}
+
+=== The plan orders dependents after the grants
+>>> [CLI] bundle plan -o json
+{
+  "resources.catalogs.pipeline_catalog": null,
+  "resources.catalogs.pipeline_catalog.grants": [
+    {
+      "node": "resources.catalogs.pipeline_catalog",
+      "label": "${resources.catalogs.pipeline_catalog.id}"
+    }
+  ],
+  "resources.pipelines.load": [
+    {
+      "node": "resources.schemas.src",
+      "label": "${resources.schemas.src.catalog_name}"
+    },
+    {
+      "node": "resources.schemas.src",
+      "label": "${resources.schemas.src.name}"
+    },
+    {
+      "node": "resources.schemas.src.grants",
+      "label": "depends_on"
+    }
+  ],
+  "resources.schemas.src": [
+    {
+      "node": "resources.catalogs.pipeline_catalog",
+      "label": "${resources.catalogs.pipeline_catalog.name}"
+    },
+    {
+      "node": "resources.catalogs.pipeline_catalog.grants",
+      "label": "depends_on"
+    }
+  ],
+  "resources.schemas.src.grants": [
+    {
+      "node": "resources.schemas.src",
+      "label": "${resources.schemas.src.id}"
+    }
+  ]
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-pipeline-[UNIQUE_NAME]/default/files...
+Created catalogs.pipeline_catalog
+Created catalogs.pipeline_catalog.grants
+Created pipelines.load
+Created schemas.src
+Created schemas.src.grants
+Files: 3 uploaded, 0 deleted
+Resources: 5 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --summary //unity-catalog/catalogs //unity-catalog/schemas //unity-catalog/permissions //pipelines
+POST /api/2.1/unity-catalog/catalogs
+PATCH /api/2.1/unity-catalog/permissions/catalog/catalog_grants_pipeline_[UNIQUE_NAME]
+POST /api/2.1/unity-catalog/schemas
+PATCH /api/2.1/unity-catalog/permissions/schema/catalog_grants_pipeline_[UNIQUE_NAME].src
+POST /api/2.0/pipelines
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.pipeline_catalog
+  delete resources.pipelines.load
+  delete resources.schemas.src
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.src
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.load
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-pipeline-[UNIQUE_NAME]/default
+
+Destroy: 3 deleted

acceptance/bundle/resources/grants/schemas/pipeline_run_as/script

@@ -0,0 +1,22 @@
+# Creating the pipeline fails unless its run_as principal can already create tables in the
+# schema, so the pipeline must be deployed after the grants it relies on.
+# See https://github.com/databricks/cli/issues/4573
+export RUN_AS_SP="${TEST_SP_APPLICATION_ID:-6f1c2b3a-4d5e-4f60-8a7b-9c0d1e2f3a4b}"
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "The grants of a referenced catalog or schema become an implicit lifecycle.depends_on"
+trace $CLI bundle validate -o json | jq '.resources | map_values(map_values(.lifecycle.depends_on))'
+
+title "The plan orders dependents after the grants"
+trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)'
+
+trace $CLI bundle deploy
+trace print_requests.py --summary //unity-catalog/catalogs //unity-catalog/schemas //unity-catalog/permissions //pipelines
+
+trace $CLI bundle plan

acceptance/bundle/resources/grants/schemas/pipeline_run_as/test.toml

@@ -0,0 +1,3 @@
+# Local only: on cloud the deploying identity lacks the servicePrincipal.user role on
+# TEST_SP_APPLICATION_ID, so it cannot set that service principal as the pipeline's run_as.
+Cloud = false

acceptance/bundle/resources/grants/schemas/pipeline_run_as/transformation.py

@@ -0,0 +1 @@
+print("Hello, World!")

bundle/config/mutator/resourcemutator/capture_grants_dependencies.go

@@ -0,0 +1,172 @@
+package resourcemutator
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"reflect"
+	"slices"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
+)
+
+type captureGrantsDependencies struct{}
+
+// CaptureGrantsDependencies makes a resource that depends on another resource also depend on
+// that resource's grants, because creating some resources requires the grants to be in place:
+// a pipeline cannot be created before its run_as principal has CREATE TABLE on the target
+// schema. The dependency is added to lifecycle.depends_on, and skipped when it would create a
+// cycle, e.g. when a schema grants privileges to the service principal of an app that refers
+// to the schema.
+func CaptureGrantsDependencies() bundle.Mutator {
+	return &captureGrantsDependencies{}
+}
+
+func (m *captureGrantsDependencies) Name() string {
+	return "CaptureGrantsDependencies"
+}
+
+func (m *captureGrantsDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	graph, err := dependencyGraph(b)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// Resources are visited in a fixed order because an added dependency can rule out a later
+	// one: with two of them forming a cycle together, the same one must be skipped on every run.
+	added := make(map[string][]string)
+	for _, node := range slices.Sorted(maps.Keys(graph)) {
+		// Grants and permissions have no lifecycle.
+		if strings.Contains(config.GetResourceTypeFromKey(node), ".") {
+			continue
+		}
+
+		// The grants appended below are not visited: range evaluates the slice once.
+		for _, dep := range graph[node] {
+			grants := dep + ".grants"
+			if _, ok := graph[grants]; !ok || slices.Contains(graph[node], grants) || reaches(graph, grants, node) {
+				continue
+			}
+			graph[node] = append(graph[node], grants)
+			added[node] = append(added[node], grants)
+		}
+	}
+
+	resources := dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey())
+	err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		return dyn.MapByPattern(root, resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			grants := added[p.String()]
+			if len(grants) == 0 {
+				return v, nil
+			}
+			return appendDependsOn(v, grants)
+		})
+	})
+	return diag.FromErr(err)
+}
+
+// dependencyGraph returns, for every resource and its grants or permissions, the nodes it depends
+// on: those it refers to and those in its lifecycle.depends_on. Nodes are keyed as in the deployment
+// plan, e.g. "resources.schemas.foo.grants"; a node without dependencies maps to nil.
+func dependencyGraph(b *bundle.Bundle) (map[string][]string, error) {
+	graph := make(map[string][]string)
+
+	err := dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error {
+		node, resourceType := config.GetNodeAndType(p)
+		if resourceType == "" {
+			return nil
+		}
+
+		src := node.String()
+		if len(p) == len(node) {
+			// Grants and permissions without entries are not deployed, so nothing can depend on them.
+			if seq, ok := v.AsSequence(); len(node) == 4 && (!ok || len(seq) == 0) {
+				return nil
+			}
+			if _, ok := graph[src]; !ok {
+				graph[src] = nil
+			}
+			return nil
+		}
+
+		ref, ok := dynvar.NewRef(v)
+		if !ok {
+			return nil
+		}
+		for _, r := range ref.References() {
+			path, err := dyn.NewPathFromString(r)
+			if err != nil {
+				return fmt.Errorf("parsing %q: %w", r, err)
+			}
+			if target, targetType := config.GetNodeAndType(path); targetType != "" {
+				graph[src] = append(graph[src], target.String())
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, group := range b.Config.Resources.AllResources() {
+		for key, resource := range group.Resources {
+			// Python-generated configs can contain null resource entries.
+			if reflect.ValueOf(resource).IsNil() {
+				continue
+			}
+			src := "resources." + group.Description.PluralName + "." + key
+			for _, dep := range resource.GetLifecycle().GetDependsOn() {
+				// An entry that does not resolve is reported by validate.LifecycleDependsOn.
+				if target, err := b.Config.Resources.ResolveDependsOn(dep.Resource); err == nil {
+					graph[src] = append(graph[src], target)
+				}
+			}
+		}
+	}
+	return graph, nil
+}
+
+// reaches reports whether from is to or depends on it, directly or through other nodes.
+func reaches(graph map[string][]string, from, to string) bool {
+	visited := make(map[string]bool)
+	stack := []string{from}
+	for len(stack) > 0 {
+		n := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if n == to {
+			return true
+		}
+		if visited[n] {
+			continue
+		}
+		visited[n] = true
+		stack = append(stack, graph[n]...)
+	}
+	return false
+}
+
+// appendDependsOn appends lifecycle.depends_on entries for the given nodes to a resource.
+func appendDependsOn(resource dyn.Value, nodes []string) (dyn.Value, error) {
+	lifecycle := dyn.GetValue(resource, "lifecycle")
+	if lifecycle.Kind() != dyn.KindMap {
+		lifecycle = dyn.V(dyn.NewMapping())
+	}
+
+	entries, _ := dyn.GetValue(lifecycle, "depends_on").AsSequence()
+	for _, node := range nodes {
+		entries = append(entries, dyn.V(map[string]dyn.Value{
+			"resource": dyn.V(strings.TrimPrefix(node, "resources.")),
+		}))
+	}
+
+	lifecycle, err := dyn.Set(lifecycle, "depends_on", dyn.V(entries))
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+	return dyn.Set(resource, "lifecycle", lifecycle)
+}