diff --git a/.nextchanges/bundles/exclude-resource-selector.md b/.nextchanges/bundles/exclude-resource-selector.md new file mode 100644 index 00000000000..d6dea26b400 --- /dev/null +++ b/.nextchanges/bundles/exclude-resource-selector.md @@ -0,0 +1 @@ +* Add `--exclude` to `bundle deploy` and `bundle plan` to defer selected resources. ([#7348](https://github.com/databricks/cli/pull/7348)) diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index 2b4838b6fa6..1876129ac4f 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -15,6 +15,7 @@ Usage: Flags: --auto-approve Skip interactive approvals that might be required for deployment. -c, --cluster-id string Override cluster in the deployment with the given cluster ID. + --exclude strings Deploy all resources except the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated. --fail-on-active-runs Fail if there are running jobs or pipelines in the deployment. --force Force-override Git branch validation. --force-lock Force acquisition of deployment lock. diff --git a/acceptance/bundle/select/exclude_basic/databricks-removed.yml.tmpl b/acceptance/bundle/select/exclude_basic/databricks-removed.yml.tmpl new file mode 100644 index 00000000000..cda695d7efc --- /dev/null +++ b/acceptance/bundle/select/exclude_basic/databricks-removed.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: exclude-$UNIQUE_NAME + +resources: + jobs: + deferred: + name: deferred-$UNIQUE_NAME + tasks: + - task_key: run_prerequisite + run_job_task: + job_id: 123 diff --git a/acceptance/bundle/select/exclude_basic/databricks.yml.tmpl b/acceptance/bundle/select/exclude_basic/databricks.yml.tmpl new file mode 100644 index 00000000000..701f4e011eb --- /dev/null +++ b/acceptance/bundle/select/exclude_basic/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: exclude-$UNIQUE_NAME + +resources: + jobs: + prerequisite: + name: prerequisite-$UNIQUE_NAME + + deferred: + name: deferred-$UNIQUE_NAME + tasks: + - task_key: run_prerequisite + run_job_task: + job_id: ${resources.jobs.prerequisite.id} diff --git a/acceptance/bundle/select/exclude_basic/out.test.toml b/acceptance/bundle/select/exclude_basic/out.test.toml new file mode 100644 index 00000000000..f7fafbcc731 --- /dev/null +++ b/acceptance/bundle/select/exclude_basic/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/exclude_basic/output.txt b/acceptance/bundle/select/exclude_basic/output.txt new file mode 100644 index 00000000000..002f3f8f6e3 --- /dev/null +++ b/acceptance/bundle/select/exclude_basic/output.txt @@ -0,0 +1,96 @@ + +>>> [CLI] bundle plan --exclude jobs.deferred +create jobs.prerequisite + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged, 1 not selected + +>>> [CLI] bundle plan --exclude jobs.deferred --exclude deferred +create jobs.prerequisite + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged, 1 not selected + +>>> [CLI] bundle plan --exclude jobs.prerequisite +Error: cannot exclude jobs.prerequisite: jobs.deferred depends on it through ${resources.jobs.prerequisite.id}; also exclude jobs.deferred or remove the dependency + + +>>> [CLI] bundle deploy --exclude jobs.missing +Error: no such resource: jobs.missing + + +>>> [CLI] bundle deploy --select jobs.deferred --exclude jobs.prerequisite +Error: --select and --exclude cannot be used together + +>>> [CLI] bundle deploy --exclude jobs.deferred --plan plan.json +Error: --exclude cannot be used with --plan; generate a filtered plan with bundle plan --exclude, then apply it with --plan + +>>> env DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan --exclude jobs.deferred +Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + +Unset the DATABRICKS_BUNDLE_ENGINE environment variable (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +=== Deploy prerequisites with --exclude +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default/files... +Created jobs.prerequisite +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged, 1 not selected + +>>> [CLI] bundle summary +Name: exclude-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default +Resources: + Jobs: + deferred: + Name: deferred-[UNIQUE_NAME] + URL: (not deployed) + prerequisite: + Name: prerequisite-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/jobs/[PREREQUISITE_ID]?w=[WORKSPACE_ID] + +>>> print_requests.py --nostamp --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "prerequisite-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } +} + +=== Deploy deferred resource with --select + +>>> [CLI] bundle deploy --select jobs.deferred +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default/files... +Created jobs.deferred +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle plan --exclude jobs.deferred +Error: cannot exclude jobs.deferred: deployed jobs.deferred still depends on jobs.prerequisite through ${resources.jobs.prerequisite.id}, which is planned for delete; also exclude jobs.prerequisite or update jobs.deferred first + + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Destroy + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.deferred + delete resources.jobs.prerequisite + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/select/exclude_basic/script b/acceptance/bundle/select/exclude_basic/script new file mode 100644 index 00000000000..d797b00da2c --- /dev/null +++ b/acceptance/bundle/select/exclude_basic/script @@ -0,0 +1,34 @@ +envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Destroy" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle plan --exclude jobs.deferred +trace $CLI bundle plan --exclude jobs.deferred --exclude deferred +musterr trace $CLI bundle plan --exclude jobs.prerequisite +musterr trace $CLI bundle deploy --exclude jobs.missing +musterr trace $CLI bundle deploy --select jobs.deferred --exclude jobs.prerequisite +musterr trace $CLI bundle deploy --exclude jobs.deferred --plan plan.json +musterr trace env DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan --exclude jobs.deferred + +$CLI bundle plan --exclude jobs.deferred -o json > plan.json +title "Deploy prerequisites with --exclude" +if [ -n "$READPLAN" ]; then + $CLI bundle deploy --plan plan.json +else + $CLI bundle deploy --exclude jobs.deferred +fi +replace_ids.py +trace $CLI bundle summary +trace print_requests.py --nostamp --sort //jobs + +title "Deploy deferred resource with --select" +trace $CLI bundle deploy --select jobs.deferred +# The deployed job still references its prerequisite until it is updated. +envsubst '$UNIQUE_NAME' < databricks-removed.yml.tmpl > databricks.yml +musterr trace $CLI bundle plan --exclude jobs.deferred +envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml +trace $CLI bundle plan diff --git a/acceptance/bundle/select/exclude_basic/test.toml b/acceptance/bundle/select/exclude_basic/test.toml new file mode 100644 index 00000000000..52fd91fcfa9 --- /dev/null +++ b/acceptance/bundle/select/exclude_basic/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = ["", "1"] +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json"] diff --git a/acceptance/bundle/select/exclude_immutable_snapshot/databricks.yml.tmpl b/acceptance/bundle/select/exclude_immutable_snapshot/databricks.yml.tmpl new file mode 100644 index 00000000000..478fe86cf7d --- /dev/null +++ b/acceptance/bundle/select/exclude_immutable_snapshot/databricks.yml.tmpl @@ -0,0 +1,31 @@ +bundle: + name: exclude-immutable-$UNIQUE_NAME + +workspace: + sync_mode: immutable + +resources: + jobs: + deferred: + name: deferred-$UNIQUE_NAME + tasks: + - task_key: run + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" + + active: + name: active-$UNIQUE_NAME + tasks: + - task_key: run + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" diff --git a/acceptance/bundle/select/exclude_immutable_snapshot/out.test.toml b/acceptance/bundle/select/exclude_immutable_snapshot/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/select/exclude_immutable_snapshot/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/select/exclude_immutable_snapshot/output.txt b/acceptance/bundle/select/exclude_immutable_snapshot/output.txt new file mode 100644 index 00000000000..77e87516d02 --- /dev/null +++ b/acceptance/bundle/select/exclude_immutable_snapshot/output.txt @@ -0,0 +1,27 @@ + +=== Deploy both jobs from the first snapshot + +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.active +Created jobs.deferred +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Recreate the snapshot while excluding the deferred job + +>>> [CLI] bundle deploy --exclude jobs.deferred +Recreated internal_immutable_snapshots.immutable +Updated jobs.active +Files: 0 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged, 1 not selected +OK: excluded job keeps its existing snapshot; active job uses the new one + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.active + delete resources.jobs.deferred + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/exclude-immutable-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/select/exclude_immutable_snapshot/script b/acceptance/bundle/select/exclude_immutable_snapshot/script new file mode 100644 index 00000000000..9d37233e857 --- /dev/null +++ b/acceptance/bundle/select/exclude_immutable_snapshot/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy both jobs from the first snapshot" +trace $CLI bundle deploy +DEFERRED_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.deferred.id') +ACTIVE_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.active.id') +OLD_PATH=$($CLI jobs get "$DEFERRED_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +OLD_CONTENT_PATH="${OLD_PATH%/files/src/main.py}" + +title "Recreate the snapshot while excluding the deferred job" +env -u MSYS_NO_PATHCONV update_file.py src/main.py "Hello from" "Hello again from" +trace $CLI bundle deploy --exclude jobs.deferred +DEFERRED_PATH=$($CLI jobs get "$DEFERRED_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +ACTIVE_PATH=$($CLI jobs get "$ACTIVE_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +if [ "$DEFERRED_PATH" != "$OLD_PATH" ] || [ "$ACTIVE_PATH" = "$OLD_PATH" ]; then + echo "ERROR: jobs do not reference the expected snapshots" + exit 1 +fi +if ! $CLI workspace get-status "$OLD_CONTENT_PATH" > /dev/null; then + echo "ERROR: the excluded job's snapshot is missing" + exit 1 +fi +echo "OK: excluded job keeps its existing snapshot; active job uses the new one" diff --git a/acceptance/bundle/select/exclude_immutable_snapshot/src/main.py b/acceptance/bundle/select/exclude_immutable_snapshot/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/select/exclude_immutable_snapshot/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/select/exclude_immutable_snapshot/test.toml b/acceptance/bundle/select/exclude_immutable_snapshot/test.toml new file mode 100644 index 00000000000..190f46eccd1 --- /dev/null +++ b/acceptance/bundle/select/exclude_immutable_snapshot/test.toml @@ -0,0 +1,3 @@ +Cloud = false +Env.MSYS_NO_PATHCONV = "1" +Ignore = [".databricks", ".gitignore", "databricks.yml"] diff --git a/bundle/bundle.go b/bundle/bundle.go index 9f686a7e466..16cee7797a4 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -197,6 +197,9 @@ type Bundle struct { // When non-empty, only the specified resources are included in deployment. Select []string + // Exclude contains resources omitted from deployment. + Exclude []string + // MigratingToDirect is set when the existing state still uses terraform, so it is // migrated to the direct engine in memory before the command runs (statemgmt.Migrate). // Deploy and destroy commit the migration (statemgmt.CommitMigration); other commands diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 44d6e9f0f12..61a69a8a242 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -13,6 +13,7 @@ import ( "strings" "sync" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" @@ -33,9 +34,8 @@ type Plan struct { Plan map[string]*PlanEntry `json:"plan,omitzero"` - // NotSelected is the number of resources removed by FilterToSelected via the - // --select flag. Serialized so the summary survives a deploy from a plan file - // (--plan); used only for summary reporting. + // NotSelected is the number of resources removed by --select or --exclude. + // Serialized so the summary survives a deploy from a plan file (--plan). NotSelected int `json:"not_selected,omitempty"` mutex sync.Mutex `json:"-"` @@ -164,6 +164,9 @@ type DependsOnEntry struct { Label string `json:"label,omitempty"` } +// LifecycleDependsOnLabel identifies dependencies that only order deployment. +const LifecycleDependsOnLabel = "depends_on" + type Changes map[string]*ChangeDesc type ChangeDesc struct { @@ -314,7 +317,7 @@ func (p *Plan) ReadUnlockEntry(resourceKey string) { p.lockmap.RUnlock(resourceKey) } -// ResolveSelectors maps each --select selector to its plan key ("resources.type.name") +// ResolveSelectors maps each resource selector to its plan key ("resources.type.name") // against resourceKeys of that same form. A selector is a qualified "type.name" or an // unqualified "name"; it errors on no match or, when unqualified, more than one. // Duplicate resourceKeys (a resource present in both config and state) collapse, so an @@ -455,6 +458,75 @@ func (p *Plan) FilterToSelected(_ context.Context, selected []string) error { return nil } +// FilterToExcluded removes the named resources and their child plan nodes. It +// rejects exclusions that would break references in retained or deployed resources. +func (p *Plan) FilterToExcluded(excluded []string, existingDependencies map[string][]DependsOnEntry) error { + resolved, err := ResolveSelectors(maps.Keys(p.Plan), excluded) + if err != nil { + return err + } + + toRemove := make(map[string]struct{}, len(resolved)) + for _, key := range resolved { + toRemove[key] = struct{}{} + for child := range p.Plan { + if strings.HasPrefix(child, key+".") { + toRemove[child] = struct{}{} + } + } + } + + var conflicts []string + for key, entry := range p.Plan { + _, removed := toRemove[key] + dependencies := entry.DependsOn + if removed { + dependencies = existingDependencies[key] + } + for _, dep := range dependencies { + dependencyEntry, exists := p.Plan[dep.Node] + if !exists { + continue + } + _, dependencyRemoved := toRemove[dep.Node] + if removed == dependencyRemoved { + continue + } + if removed && dep.Label == LifecycleDependsOnLabel { + continue + } + dependent := strings.TrimPrefix(key, "resources.") + dependency := strings.TrimPrefix(dep.Node, "resources.") + reference := "" + if dep.Label != "" { + reference = " through " + dep.Label + } + if dependencyRemoved { + if entry.Action == Delete { + continue + } + conflicts = append(conflicts, fmt.Sprintf("cannot exclude %s: %s depends on it%s; also exclude %s or remove the dependency", dependency, dependent, reference, dependent)) + } else { + // Recreating an immutable snapshot leaves its old path available. + if dependencyEntry.Action.KeepsID() || (dep.Node == resources.SnapshotKey && dependencyEntry.Action == Recreate) { + continue + } + conflicts = append(conflicts, fmt.Sprintf("cannot exclude %s: deployed %s still depends on %s%s, which is planned for %s; also exclude %s or update %s first", dependent, dependent, dependency, reference, dependencyEntry.Action, dependency, dependent)) + } + } + } + if len(conflicts) > 0 { + slices.Sort(conflicts) + return errors.New(strings.Join(conflicts, "\n")) + } + + for key := range toRemove { + delete(p.Plan, key) + } + p.NotSelected = len(toRemove) + return nil +} + // reverseEdges returns the reverse of the plan's DependsOn edges: for each node, the // nodes that depend on it. func (p *Plan) reverseEdges() map[string][]string { diff --git a/bundle/deployplan/plan_filter_test.go b/bundle/deployplan/plan_filter_test.go index 8c6dd30240e..3826f4fe13d 100644 --- a/bundle/deployplan/plan_filter_test.go +++ b/bundle/deployplan/plan_filter_test.go @@ -283,3 +283,133 @@ func TestFilterToSelected_DeletionLeavesDependencies(t *testing.T) { assert.Contains(t, p.Plan, "resources.jobs.gone") assert.NotContains(t, p.Plan, "resources.jobs.lib") } + +func TestFilterToExcluded_LeavesOtherResourcesAndDeduplicatesSelectors(t *testing.T) { + p := planWithDeps() + require.NoError(t, p.FilterToExcluded([]string{"jobs.independent", "independent", "jobs.independent"}, nil)) + assert.NotContains(t, p.Plan, "resources.jobs.independent") + assert.Contains(t, p.Plan, "resources.jobs.foo") + assert.Contains(t, p.Plan, "resources.jobs.bar") + assert.Contains(t, p.Plan, "resources.jobs.baz") + assert.Equal(t, 1, p.NotSelected) +} + +func TestFilterToExcluded_RemovesGrantsAndPermissions(t *testing.T) { + p := planWithGrants() + require.NoError(t, p.FilterToExcluded([]string{"schemas.bronze"}, nil)) + assert.NotContains(t, p.Plan, "resources.schemas.bronze") + assert.NotContains(t, p.Plan, "resources.schemas.bronze.grants") + assert.NotContains(t, p.Plan, "resources.schemas.bronze.permissions") + assert.Contains(t, p.Plan, "resources.schemas.silver") + assert.Contains(t, p.Plan, "resources.schemas.silver.grants") + assert.Equal(t, 3, p.NotSelected) +} + +func TestFilterToExcluded_RejectsRequiredDependencyWithoutChangingPlan(t *testing.T) { + p := planWithDeps() + p.Plan["resources.jobs.foo"].DependsOn[0].Label = "${resources.jobs.bar.id}" + err := p.FilterToExcluded([]string{"jobs.bar"}, nil) + require.Error(t, err) + assert.ErrorContains(t, err, "cannot exclude jobs.bar: jobs.foo depends on it") + assert.ErrorContains(t, err, "${resources.jobs.bar.id}") + assert.Len(t, p.Plan, 4) + assert.Zero(t, p.NotSelected) +} + +func TestFilterToExcluded_RejectsUnknownAndAmbiguousSelectors(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.same"] = &deployplan.PlanEntry{} + p.Plan["resources.pipelines.same"] = &deployplan.PlanEntry{} + assert.ErrorContains(t, p.FilterToExcluded([]string{"unknown"}, nil), "no such resource: unknown") + assert.ErrorContains(t, p.FilterToExcluded([]string{"same"}, nil), "ambiguous resource: same") + assert.Len(t, p.Plan, 2) +} + +func TestFilterToExcluded_OmitsSelectedDeletion(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.keep"] = &deployplan.PlanEntry{Action: deployplan.Create} + p.Plan["resources.jobs.gone"] = &deployplan.PlanEntry{Action: deployplan.Delete} + require.NoError(t, p.FilterToExcluded([]string{"jobs.gone"}, nil)) + assert.NotContains(t, p.Plan, "resources.jobs.gone") + assert.Equal(t, deployplan.Create, p.Plan["resources.jobs.keep"].Action) + assert.Equal(t, 1, p.NotSelected) +} + +func TestFilterToExcluded_RejectsIDChangesNeededByExcludedResource(t *testing.T) { + for _, action := range []deployplan.ActionType{deployplan.Delete, deployplan.Recreate, deployplan.UpdateWithID} { + t.Run(string(action), func(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.bar"] = &deployplan.PlanEntry{Action: action} + p.Plan["resources.jobs.foo"] = &deployplan.PlanEntry{Action: deployplan.Update} + existingDependencies := map[string][]deployplan.DependsOnEntry{ + "resources.jobs.foo": { + {Node: "resources.jobs.bar", Label: deployplan.LifecycleDependsOnLabel}, + {Node: "resources.jobs.bar", Label: "${resources.jobs.bar.id}"}, + }, + } + + err := p.FilterToExcluded([]string{"jobs.foo"}, existingDependencies) + require.Error(t, err) + assert.ErrorContains(t, err, "deployed jobs.foo still depends on jobs.bar through ${resources.jobs.bar.id}") + assert.ErrorContains(t, err, "planned for "+string(action)) + assert.Len(t, p.Plan, 2) + assert.Zero(t, p.NotSelected) + }) + } +} + +func TestFilterToExcluded_AllowsStableDependencyOfExcludedResource(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.bar"] = &deployplan.PlanEntry{Action: deployplan.Update} + p.Plan["resources.jobs.foo"] = &deployplan.PlanEntry{Action: deployplan.Update} + existingDependencies := map[string][]deployplan.DependsOnEntry{ + "resources.jobs.foo": {{Node: "resources.jobs.bar"}}, + } + + require.NoError(t, p.FilterToExcluded([]string{"jobs.foo"}, existingDependencies)) + assert.NotContains(t, p.Plan, "resources.jobs.foo") + assert.Contains(t, p.Plan, "resources.jobs.bar") +} + +func TestFilterToExcluded_AllowsRecreatedImmutableSnapshot(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.internal_immutable_snapshots.immutable"] = &deployplan.PlanEntry{Action: deployplan.Recreate} + p.Plan["resources.jobs.deferred"] = &deployplan.PlanEntry{Action: deployplan.Update} + existingDependencies := map[string][]deployplan.DependsOnEntry{ + "resources.jobs.deferred": {{Node: "resources.internal_immutable_snapshots.immutable", Label: "${resources.internal_immutable_snapshots.immutable.full_path}"}}, + } + + require.NoError(t, p.FilterToExcluded([]string{"jobs.deferred"}, existingDependencies)) + assert.Contains(t, p.Plan, "resources.internal_immutable_snapshots.immutable") + assert.NotContains(t, p.Plan, "resources.jobs.deferred") +} + +func TestFilterToExcluded_AllowsChangingOrderingDependency(t *testing.T) { + for _, action := range []deployplan.ActionType{deployplan.Delete, deployplan.Recreate, deployplan.UpdateWithID} { + t.Run(string(action), func(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.prerequisite"] = &deployplan.PlanEntry{Action: action} + p.Plan["resources.genie_spaces.deferred"] = &deployplan.PlanEntry{Action: deployplan.Skip} + existingDependencies := map[string][]deployplan.DependsOnEntry{ + "resources.genie_spaces.deferred": {{Node: "resources.jobs.prerequisite", Label: deployplan.LifecycleDependsOnLabel}}, + } + + require.NoError(t, p.FilterToExcluded([]string{"genie_spaces.deferred"}, existingDependencies)) + assert.Contains(t, p.Plan, "resources.jobs.prerequisite") + assert.NotContains(t, p.Plan, "resources.genie_spaces.deferred") + assert.Equal(t, 1, p.NotSelected) + }) + } +} + +func TestFilterToExcluded_AllowsDeletingDependentOfExcludedResource(t *testing.T) { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.bar"] = &deployplan.PlanEntry{Action: deployplan.Skip} + p.Plan["resources.jobs.foo"] = &deployplan.PlanEntry{ + Action: deployplan.Delete, + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.bar"}}, + } + require.NoError(t, p.FilterToExcluded([]string{"jobs.bar"}, nil)) + assert.NotContains(t, p.Plan, "resources.jobs.bar") + assert.Equal(t, deployplan.Delete, p.Plan["resources.jobs.foo"].Action) +} diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 43a58291ab2..baff30329e8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1775,7 +1775,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // (e.g. skipped as empty) needs no ordering. for _, entry := range p.Plan { entry.DependsOn = slices.DeleteFunc(entry.DependsOn, func(dep deployplan.DependsOnEntry) bool { - return dep.Label == lifecycleDependsOnLabel && p.Plan[dep.Node] == nil + return dep.Label == deployplan.LifecycleDependsOnLabel && p.Plan[dep.Node] == nil }) } @@ -1879,8 +1879,6 @@ func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Ada return adapter, nil } -const lifecycleDependsOnLabel = "depends_on" - // LifecycleDependsOn returns the edges declared in lifecycle.depends_on of a resource, // given its config as returned by [config.Root.GetResourceConfig]. func LifecycleDependsOn(configRoot *config.Root, inputConfig any) ([]deployplan.DependsOnEntry, error) { @@ -1898,7 +1896,7 @@ func LifecycleDependsOn(configRoot *config.Root, inputConfig any) ([]deployplan. if err != nil { return nil, fmt.Errorf("lifecycle.depends_on: %w", err) } - result = append(result, deployplan.DependsOnEntry{Node: target, Label: lifecycleDependsOnLabel}) + result = append(result, deployplan.DependsOnEntry{Node: target, Label: deployplan.LifecycleDependsOnLabel}) } return result, nil } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 1f6831281ea..b92386b3a17 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -134,8 +134,8 @@ func logDeploySummary(ctx context.Context, b *bundle.Bundle, plan *deployplan.Pl counts := plan.CountActions() summary := fmt.Sprintf("Resources: %d created, %d changed, %d deleted, %d unchanged", counts.Create, counts.Change, counts.Delete, counts.Unchanged) // Gate on the plan's own NotSelected (not b.Select) so the suffix survives a - // deploy from a --plan file, where --select was applied at plan time and - // b.Select is empty here. NotSelected is only ever set by FilterToSelected. + // deploy from a --plan file, where filtering was applied at plan time and + // b.Select and b.Exclude are empty here. if plan.NotSelected > 0 { summary += fmt.Sprintf(", %d not selected", plan.NotSelected) } @@ -366,6 +366,17 @@ func RunPlan(ctx context.Context, b *bundle.Bundle) *deployplan.Plan { logdiag.LogError(ctx, err) return nil } + } else if len(b.Exclude) > 0 { + existingDependencies := make(map[string][]deployplan.DependsOnEntry) + for key := range plan.Plan { + if entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry(key); ok { + existingDependencies[key] = entry.DependsOn + } + } + if err := plan.FilterToExcluded(b.Exclude, existingDependencies); err != nil { + logdiag.LogError(ctx, err) + return nil + } } return plan } diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 571b822a192..d38a1e62d01 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -33,6 +33,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa var quiet int var readPlanPath string var selectResources []string + var excludeResources []string cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") @@ -44,6 +45,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa cmd.Flags().CountVarP(&quiet, "quiet", "q", "Reduce output: -q prints only the summary, -qq prints only warnings and errors.") cmd.Flags().StringVar(&readPlanPath, "plan", "", "Path to a JSON plan file to apply instead of planning.") cmd.Flags().StringSliceVar(&selectResources, "select", nil, "Deploy only the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.") + cmd.Flags().StringSliceVar(&excludeResources, "exclude", nil, "Deploy all resources except the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.") // Verbose flag currently only affects file sync output, it's used by the vscode extension cmd.Flags().MarkHidden("verbose") @@ -54,6 +56,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa utils.SetForceLock(cmd, b, forceLock) b.AutoApprove = autoApprove b.Select = selectResources + b.Exclude = excludeResources b.Quiet = bundle.QuietLevel(quiet) if cmd.Flag("compute-id").Changed { diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index b61208eb6de..0e57e3944dd 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -31,12 +31,14 @@ It is useful for previewing changes before running 'bundle deploy'.`, var quiet int var clusterId string var selectResources []string + var excludeResources []string cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().CountVarP(&quiet, "quiet", "q", "Reduce output: -q prints only the summary, -qq prints nothing on success.") cmd.Flags().StringVar(&clusterId, "compute-id", "", "Override cluster in the deployment with the given compute ID.") cmd.Flags().StringVarP(&clusterId, "cluster-id", "c", "", "Override cluster in the deployment with the given cluster ID.") cmd.Flags().MarkDeprecated("compute-id", "use --cluster-id instead") cmd.Flags().StringSliceVar(&selectResources, "select", nil, "Plan only the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.") + cmd.Flags().StringSliceVar(&excludeResources, "exclude", nil, "Plan all resources except the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.") cmd.RunE = func(cmd *cobra.Command, args []string) error { opts := utils.ProcessOptions{ @@ -47,6 +49,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, InitFunc: func(b *bundle.Bundle) { b.Config.Bundle.Force = force b.Select = selectResources + b.Exclude = excludeResources if cmd.Flag("compute-id").Changed { b.Config.Bundle.ClusterId = clusterId @@ -104,7 +107,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, } // Note, this string should not be changed, "bundle deployment migrate" depends on this format: fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged", counts.Create, counts.Change, counts.Delete, counts.Unchanged) - if len(selectResources) > 0 { + if len(selectResources) > 0 || len(excludeResources) > 0 { fmt.Fprintf(out, ", %d not selected", plan.NotSelected) } fmt.Fprintln(out) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 68798a4bb01..9b2870627d0 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -106,10 +106,9 @@ func ProcessBundle(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, err return b, err } -// validateSelectedResources rejects a --select selector that names no resource in -// config or state. Deletion targets live only in state, so it must run after the -// state is open; ResolveSelectors collapses the config/state key overlap. -func validateSelectedResources(ctx context.Context, b *bundle.Bundle) error { +// validateResourceSelectors rejects selectors that name no resource in config +// or state. Deletion targets live only in state, so it runs after state is open. +func validateResourceSelectors(ctx context.Context, b *bundle.Bundle) error { var keys []string for _, group := range b.Config.Resources.AllResources() { for name := range group.Resources { @@ -120,6 +119,10 @@ func validateSelectedResources(ctx context.Context, b *bundle.Bundle) error { keys = append(keys, key) } _, err := deployplan.ResolveSelectors(slices.Values(keys), b.Select) + if err != nil { + return err + } + _, err = deployplan.ResolveSelectors(slices.Values(keys), b.Exclude) return err } @@ -175,6 +178,12 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.InitFunc != nil { bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) { opts.InitFunc(b) }) } + if len(b.Select) > 0 && len(b.Exclude) > 0 { + return b, nil, errors.New("--select and --exclude cannot be used together") + } + if opts.ReadPlanPath != "" && len(b.Exclude) > 0 { + return b, nil, errors.New("--exclude cannot be used with --plan; generate a filtered plan with bundle plan --exclude, then apply it with --plan") + } // InitFunc is where -q is applied, so the quiet context can only be derived // afterwards. Progress messages are emitted from mutators that receive only a @@ -501,15 +510,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - // Reject an unknown --select before PreDeployChecks and the predeploy script + // Reject an unknown selector before PreDeployChecks and the predeploy script // (and well before deploy uploads anything). Deletion targets live only in state, // so this is the earliest point a selector can be checked. Note: for a valid // deletion target whose live dependent makes it unsafe to delete, the predeploy - // script still runs before FilterToSelected (in RunPlan) catches the conflict; + // script still runs before plan filtering (in RunPlan) catches the conflict; // that case requires the plan to detect it. Skipped for a saved plan (plan != nil); - // for a fresh plan, FilterToSelected re-validates and filters it. - if (opts.Deploy || opts.PreDeployChecks) && plan == nil && len(b.Select) > 0 { - if err := validateSelectedResources(ctx, b); err != nil { + // for a fresh plan, the filter re-validates the selectors. + if (opts.Deploy || opts.PreDeployChecks) && plan == nil && (len(b.Select) > 0 || len(b.Exclude) > 0) { + if err := validateResourceSelectors(ctx, b); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/docs/direct.md b/docs/direct.md index 363d9561bb1..44dda441bd0 100644 --- a/docs/direct.md +++ b/docs/direct.md @@ -62,6 +62,12 @@ New bundles use the direct engine, so no configuration is needed. The migrate co Setting `bundle.engine` (or `targets.*.engine`) or `DATABRICKS_BUNDLE_ENGINE` to "terraform" is an error. To keep using Terraform, use Databricks CLI v1.19.x. +### Deploying all but selected resources + +Use `databricks bundle deploy --exclude genie_spaces.my_space` to deploy the other resources first, then `databricks bundle deploy --select genie_spaces.my_space` when its prerequisites are ready. `databricks bundle plan --exclude genie_spaces.my_space` previews the first stage. You can save that filtered plan with `-o json > plan.json` and apply it with `databricks bundle deploy --plan plan.json`. + +Both selectors accept a unique resource name or a qualified `type.name`, and can be repeated or comma-separated. Unknown or ambiguous names fail. Excluding a resource also excludes its grants and permissions. If a resource kept in the plan depends on an excluded resource, the command fails with both names, even when the retained resource is unchanged. Deleting or replacing a resource still referenced by an excluded deployed resource fails when the old reference would become invalid, based on the recorded state dependencies. Recreating an immutable bundle snapshot keeps its old path available, so excluded jobs can keep using it while other jobs move to the new snapshot. Ordering-only `lifecycle.depends_on` entries do not retain a prerequisite's ID when their dependent resource is excluded. `--exclude` cannot be combined with `--select` or passed when applying a saved `--plan`. + ## Differences from terraform ### Diff calculation