[draft] Add bundle deploy --exclude resource selector

.nextchanges/bundles/exclude-resource-selector.md

@@ -0,0 +1 @@
+* Add `--exclude` to `bundle deploy` and `bundle plan` to defer selected resources. ([#7348](https://github.com/databricks/cli/pull/7348))

acceptance/bundle/help/bundle-deploy/output.txt

@@ -15,6 +15,7 @@ Usage:
 Flags:
       --auto-approve          Skip interactive approvals that might be required for deployment.
   -c, --cluster-id string     Override cluster in the deployment with the given cluster ID.
+      --exclude strings       Deploy all resources except the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.
       --fail-on-active-runs   Fail if there are running jobs or pipelines in the deployment.
       --force                 Force-override Git branch validation.
       --force-lock            Force acquisition of deployment lock.

acceptance/bundle/select/exclude_basic/databricks-removed.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: exclude-$UNIQUE_NAME
+
+resources:
+  jobs:
+    deferred:
+      name: deferred-$UNIQUE_NAME
+      tasks:
+        - task_key: run_prerequisite
+          run_job_task:
+            job_id: 123

acceptance/bundle/select/exclude_basic/databricks.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: exclude-$UNIQUE_NAME
+
+resources:
+  jobs:
+    prerequisite:
+      name: prerequisite-$UNIQUE_NAME
+
+    deferred:
+      name: deferred-$UNIQUE_NAME
+      tasks:
+        - task_key: run_prerequisite
+          run_job_task:
+            job_id: ${resources.jobs.prerequisite.id}

acceptance/bundle/select/exclude_basic/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]
+EnvMatrix.READPLAN = ["", "1"]

acceptance/bundle/select/exclude_basic/output.txt

@@ -0,0 +1,96 @@
+
+>>> [CLI] bundle plan --exclude jobs.deferred
+create jobs.prerequisite
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged, 1 not selected
+
+>>> [CLI] bundle plan --exclude jobs.deferred --exclude deferred
+create jobs.prerequisite
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged, 1 not selected
+
+>>> [CLI] bundle plan --exclude jobs.prerequisite
+Error: cannot exclude jobs.prerequisite: jobs.deferred depends on it through ${resources.jobs.prerequisite.id}; also exclude jobs.deferred or remove the dependency
+
+
+>>> [CLI] bundle deploy --exclude jobs.missing
+Error: no such resource: jobs.missing
+
+
+>>> [CLI] bundle deploy --select jobs.deferred --exclude jobs.prerequisite
+Error: --select and --exclude cannot be used together
+
+>>> [CLI] bundle deploy --exclude jobs.deferred --plan plan.json
+Error: --exclude cannot be used with --plan; generate a filtered plan with bundle plan --exclude, then apply it with --plan
+
+>>> env DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan --exclude jobs.deferred
+Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0
+
+Unset the DATABRICKS_BUNDLE_ENGINE environment variable (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x.
+See https://docs.databricks.com/dev-tools/bundles/direct for details
+
+=== Deploy prerequisites with --exclude
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default/files...
+Created jobs.prerequisite
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged, 1 not selected
+
+>>> [CLI] bundle summary
+Name: exclude-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default
+Resources:
+  Jobs:
+    deferred:
+      Name: deferred-[UNIQUE_NAME]
+      URL:  (not deployed)
+    prerequisite:
+      Name: prerequisite-[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/jobs/[PREREQUISITE_ID]?w=[WORKSPACE_ID]
+
+>>> print_requests.py --nostamp --sort //jobs
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "prerequisite-[UNIQUE_NAME]",
+    "queue": {
+      "enabled": true
+    }
+  }
+}
+
+=== Deploy deferred resource with --select
+
+>>> [CLI] bundle deploy --select jobs.deferred
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default/files...
+Created jobs.deferred
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> [CLI] bundle plan --exclude jobs.deferred
+Error: cannot exclude jobs.deferred: deployed jobs.deferred still depends on jobs.prerequisite through ${resources.jobs.prerequisite.id}, which is planned for delete; also exclude jobs.prerequisite or update jobs.deferred first
+
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+=== Destroy
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.deferred
+  delete resources.jobs.prerequisite
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/exclude-[UNIQUE_NAME]/default
+
+Destroy: 2 deleted

acceptance/bundle/select/exclude_basic/script

@@ -0,0 +1,34 @@
+envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    title "Destroy"
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle plan --exclude jobs.deferred
+trace $CLI bundle plan --exclude jobs.deferred --exclude deferred
+musterr trace $CLI bundle plan --exclude jobs.prerequisite
+musterr trace $CLI bundle deploy --exclude jobs.missing
+musterr trace $CLI bundle deploy --select jobs.deferred --exclude jobs.prerequisite
+musterr trace $CLI bundle deploy --exclude jobs.deferred --plan plan.json
+musterr trace env DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan --exclude jobs.deferred
+
+$CLI bundle plan --exclude jobs.deferred -o json > plan.json
+title "Deploy prerequisites with --exclude"
+if [ -n "$READPLAN" ]; then
+    $CLI bundle deploy --plan plan.json
+else
+    $CLI bundle deploy --exclude jobs.deferred
+fi
+replace_ids.py
+trace $CLI bundle summary
+trace print_requests.py --nostamp --sort //jobs
+
+title "Deploy deferred resource with --select"
+trace $CLI bundle deploy --select jobs.deferred
+# The deployed job still references its prerequisite until it is updated.
+envsubst '$UNIQUE_NAME' < databricks-removed.yml.tmpl > databricks.yml
+musterr trace $CLI bundle plan --exclude jobs.deferred
+envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml
+trace $CLI bundle plan

acceptance/bundle/select/exclude_basic/test.toml

@@ -0,0 +1,4 @@
+Cloud = false
+RecordRequests = true
+EnvMatrix.READPLAN = ["", "1"]
+Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json"]

acceptance/bundle/select/exclude_immutable_snapshot/databricks.yml.tmpl

@@ -0,0 +1,31 @@
+bundle:
+  name: exclude-immutable-$UNIQUE_NAME
+
+workspace:
+  sync_mode: immutable
+
+resources:
+  jobs:
+    deferred:
+      name: deferred-$UNIQUE_NAME
+      tasks:
+        - task_key: run
+          spark_python_task:
+            python_file: ./src/main.py
+          environment_key: env
+      environments:
+        - environment_key: env
+          spec:
+            environment_version: "4"
+
+    active:
+      name: active-$UNIQUE_NAME
+      tasks:
+        - task_key: run
+          spark_python_task:
+            python_file: ./src/main.py
+          environment_key: env
+      environments:
+        - environment_key: env
+          spec:
+            environment_version: "4"

acceptance/bundle/select/exclude_immutable_snapshot/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/select/exclude_immutable_snapshot/output.txt

@@ -0,0 +1,27 @@
+
+=== Deploy both jobs from the first snapshot
+
+>>> [CLI] bundle deploy
+Created internal_immutable_snapshots.immutable
+Created jobs.active
+Created jobs.deferred
+Files: 0 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Recreate the snapshot while excluding the deferred job
+
+>>> [CLI] bundle deploy --exclude jobs.deferred
+Recreated internal_immutable_snapshots.immutable
+Updated jobs.active
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 1 changed, 1 deleted, 0 unchanged, 1 not selected
+OK: excluded job keeps its existing snapshot; active job uses the new one
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.active
+  delete resources.jobs.deferred
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/exclude-immutable-[UNIQUE_NAME]/default
+
+Destroy: 2 deleted

acceptance/bundle/select/exclude_immutable_snapshot/script

@@ -0,0 +1,28 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy both jobs from the first snapshot"
+trace $CLI bundle deploy
+DEFERRED_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.deferred.id')
+ACTIVE_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.active.id')
+OLD_PATH=$($CLI jobs get "$DEFERRED_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file')
+OLD_CONTENT_PATH="${OLD_PATH%/files/src/main.py}"
+
+title "Recreate the snapshot while excluding the deferred job"
+env -u MSYS_NO_PATHCONV update_file.py src/main.py "Hello from" "Hello again from"
+trace $CLI bundle deploy --exclude jobs.deferred
+DEFERRED_PATH=$($CLI jobs get "$DEFERRED_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file')
+ACTIVE_PATH=$($CLI jobs get "$ACTIVE_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file')
+if [ "$DEFERRED_PATH" != "$OLD_PATH" ] || [ "$ACTIVE_PATH" = "$OLD_PATH" ]; then
+    echo "ERROR: jobs do not reference the expected snapshots"
+    exit 1
+fi
+if ! $CLI workspace get-status "$OLD_CONTENT_PATH" > /dev/null; then
+    echo "ERROR: the excluded job's snapshot is missing"
+    exit 1
+fi
+echo "OK: excluded job keeps its existing snapshot; active job uses the new one"

acceptance/bundle/select/exclude_immutable_snapshot/src/main.py

@@ -0,0 +1 @@
+print("Hello from Spark Python Task!")

acceptance/bundle/select/exclude_immutable_snapshot/test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+Env.MSYS_NO_PATHCONV = "1"
+Ignore = [".databricks", ".gitignore", "databricks.yml"]

bundle/bundle.go

@@ -197,6 +197,9 @@ type Bundle struct {
 	// When non-empty, only the specified resources are included in deployment.
 	Select []string
 
+	// Exclude contains resources omitted from deployment.
+	Exclude []string
+
 	// MigratingToDirect is set when the existing state still uses terraform, so it is
 	// migrated to the direct engine in memory before the command runs (statemgmt.Migrate).
 	// Deploy and destroy commit the migration (statemgmt.CommitMigration); other commands