diff --git a/.nextchanges/bundles/catalog-renamed-out-of-band.md b/.nextchanges/bundles/catalog-renamed-out-of-band.md new file mode 100644 index 00000000000..1f582e5c5e5 --- /dev/null +++ b/.nextchanges/bundles/catalog-renamed-out-of-band.md @@ -0,0 +1 @@ +* Find a catalog renamed outside the bundle by its id. ([#7418](https://github.com/databricks/cli/pull/7418)) diff --git a/acceptance/bundle/refschema/output/catalogs.txt b/acceptance/bundle/refschema/output/catalogs.txt index 42fb0f78ead..df74c0d7910 100644 --- a/acceptance/bundle/refschema/output/catalogs.txt +++ b/acceptance/bundle/refschema/output/catalogs.txt @@ -11,7 +11,7 @@ resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type resources.catalogs.*.effective_predictive_optimization_flag.value catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.full_name string REMOTE -resources.catalogs.*.id string INPUT +resources.catalogs.*.id string ALL resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.depends_on []resources.DependsOn INPUT diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt index 5da6c37fe2a..64b652fe1d7 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt @@ -17,6 +17,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "action": "update", "remote": "set outside the bundle" }, + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt index 886246cec10..cfbb45cb47d 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt @@ -16,6 +16,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan --output json { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt index c37befe061b..4f03b1b9c15 100644 --- a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt +++ b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt @@ -80,6 +80,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "reason": "empty", "remote": 0 }, + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 5bb6c4b2655..12894db3bbb 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -65,10 +65,17 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "name": { "action": "update_id", "reason": "id_changes", diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index af5dcff5840..9e9f3e62aa0 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -34,23 +34,41 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml name: myvolume name: myvolume2 -=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +=== The plan finds the catalog, the schema and the volume at their new names by their ids >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "action": "create", + "id": "test_catalog_renamed_[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "name": "test_catalog_renamed_[UNIQUE_NAME]" } }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME]", + "id": "[CATALOG_ID]", + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[CATALOG_ID]", + "remote": "[CATALOG_ID]" + }, "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]" + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" } } }, @@ -73,7 +91,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", "name": "myschema2", @@ -127,7 +145,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][1], + "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", "name": "myvolume2", @@ -168,26 +186,48 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== The catalog's create fails because the renamed catalog has its name +=== Deploy only records the new names in the state ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs -HTTP Status: 400 Bad Request -API error_code: CATALOG_ALREADY_EXISTS -API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists +>>> print_requests.py //unity-catalog -Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2" + } +} -Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 +=== Redeploy is a no-op -Files: 3 uploaded, 0 deleted +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 -Destroy: 0 deleted +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default ->>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index aa51aadefb6..a0ce8b2754c 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -5,8 +5,9 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace $CLI catalogs delete "$NEW_CATALOG" --force + # A run that fails before the plan finds the renamed catalog leaves it where destroy does not + # look. Normally destroy has deleted it already, so this fails quietly. + errcode $CLI catalogs delete "$NEW_CATALOG" --force &> LOG.cleanup_catalog # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -15,12 +16,14 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location } title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests +# The SDK drops the catalog's id, so read it with a raw GET. +add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID @@ -34,11 +37,14 @@ title "Rename all three in the config to match" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ trace update_file.py databricks.yml "name: myschema" "name: myschema2" trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" -NEW_SCHEMA="$NEW_CATALOG.myschema2" -NEW_VOLUME="$NEW_SCHEMA.myvolume2" -title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +title "The plan finds the catalog, the schema and the volume at their new names by their ids" plan -title "The catalog's create fails because the renamed catalog has its name" -trace musterr $CLI bundle deploy +title "Deploy only records the new names in the state" +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml index 698c1c6ca49..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -1,8 +1,14 @@ -Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index cf94276e0cd..75997e56b47 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -20,23 +20,41 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +=== The plan finds the catalog, the schema and the volume at their new names by their ids >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "action": "create", + "id": "test_catalog_renamed_[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "name": "test_catalog_renamed_[UNIQUE_NAME]" } }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME]", + "id": "[CATALOG_ID]", + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[CATALOG_ID]", + "remote": "[CATALOG_ID]" + }, "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]" + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" } } }, @@ -59,7 +77,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", "name": "myschema", @@ -106,7 +124,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][1], + "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", "name": "myvolume", @@ -133,26 +151,48 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== The catalog's create fails because the renamed catalog has its name +=== Deploy only records the new names in the state ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs -HTTP Status: 400 Bad Request -API error_code: CATALOG_ALREADY_EXISTS -API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists +>>> print_requests.py //unity-catalog -Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume" + } +} -Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 +=== Redeploy is a no-op -Files: 3 uploaded, 0 deleted +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 -Destroy: 0 deleted +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default ->>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index 1c39637d66b..ad34b43a367 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -5,8 +5,9 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace $CLI catalogs delete "$NEW_CATALOG" --force + # A run that fails before the plan finds the renamed catalog leaves it where destroy does not + # look. Normally destroy has deleted it already, so this fails quietly. + errcode $CLI catalogs delete "$NEW_CATALOG" --force &> LOG.cleanup_catalog # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -15,12 +16,14 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location } title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests +# The SDK drops the catalog's id, so read it with a raw GET. +add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID @@ -30,11 +33,14 @@ print_requests.py //unity-catalog &> LOG.rename_requests title "Rename the catalog in the config to match" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -NEW_SCHEMA="$NEW_CATALOG.myschema" -NEW_VOLUME="$NEW_SCHEMA.myvolume" -title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +title "The plan finds the catalog, the schema and the volume at their new names by their ids" plan -title "The catalog's create fails because the renamed catalog has its name" -trace musterr $CLI bundle deploy +title "Deploy only records the new names in the state" +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml index 698c1c6ca49..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -1,8 +1,14 @@ -Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.json index 92d227bc232..49cd77a9705 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.json @@ -1,7 +1,7 @@ { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 1, "plan": { "resources.catalogs.grants_catalog": { @@ -10,8 +10,17 @@ "catalog_type": "MANAGED_CATALOG", "comment": "Test catalog for grants", "full_name": "catalog_grants_[UNIQUE_NAME]", + "id": "[UUID][1]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][1]", + "remote": "[UUID][1]" + } } }, "resources.catalogs.grants_catalog.grants": { diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt index 895e3dce8a8..5e38a51622e 100644 --- a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt @@ -30,10 +30,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + }, "name": { "action": "update_id", "reason": "id_changes", @@ -124,7 +131,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "catalog_name": { @@ -144,8 +151,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } } diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 6cac5f13385..3d886f935f3 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -49,8 +49,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } } }, "resources.schemas.schema1": { @@ -74,7 +83,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "name": { @@ -87,8 +96,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, @@ -225,10 +234,17 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + }, "name": { "action": "update_id", "reason": "id_changes", @@ -262,7 +278,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema2", "name": "myschema2", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "catalog_name": { @@ -285,8 +301,8 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index c292b659090..1968278597b 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -23,8 +23,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } } }, "resources.schemas.schema1": { @@ -48,7 +57,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "name": { @@ -61,8 +70,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt index f5d7df1a699..621819f3180 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -31,10 +31,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "name": { "action": "update_id", "reason": "id_changes", diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 53d9b3b23f1..933c580c2ba 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -3,23 +3,52 @@ package dresources import ( "context" "fmt" + "maps" + "net/http" "slices" + "sync" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +const catalogsPath = "/api/2.1/unity-catalog/catalogs" + +// CatalogRemote is CatalogInfo plus the catalog's id, a UUID that UC returns but the API spec does +// not declare, so the SDK drops it. A rename keeps the id and a recreate changes it (verified on +// AWS, Azure and GCP). +type CatalogRemote struct { + catalog.CatalogInfo + + Id string `json:"id,omitempty"` +} + +// Custom marshaler needed because embedded CatalogInfo has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *CatalogRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s CatalogRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // CatalogState is CreateCatalog plus isolation_mode, which the Create API does not -// accept; DoCreate applies it with a follow-up update. +// accept (DoCreate applies it with a follow-up update), and the catalog's id. type CatalogState struct { catalog.CreateCatalog IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + + // Recorded by the framework from the remote state (internal_id_field). + Id string `json:"id,omitempty"` } -// Custom marshalers needed because embedded catalog.CreateCatalog has its own MarshalJSON +// Custom marshaler needed because embedded CreateCatalog has its own MarshalJSON // which would otherwise take over and ignore the additional fields. func (s *CatalogState) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) @@ -30,21 +59,48 @@ func (s CatalogState) MarshalJSON() ([]byte, error) { } type ResourceCatalog struct { - client *databricks.WorkspaceClient + client *databricks.WorkspaceClient + apiClient func() (*client.DatabricksClient, error) } -func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog { - return &ResourceCatalog{client: client} +func (*ResourceCatalog) New(w *databricks.WorkspaceClient) *ResourceCatalog { + return &ResourceCatalog{ + client: w, + apiClient: sync.OnceValues(func() (*client.DatabricksClient, error) { + return client.New(w.Config) + }), + } +} + +// do calls the catalogs API like the SDK does, but decodes the response into CatalogRemote: +// the SDK's CatalogInfo has no id. +func (r *ResourceCatalog) do(ctx context.Context, method, path string, request any) (*CatalogRemote, error) { + apiClient, err := r.apiClient() + if err != nil { + return nil, err + } + headers := map[string]string{"Accept": "application/json"} + if request != nil { + headers["Content-Type"] = "application/json" + } + maps.Copy(headers, auth.WorkspaceIDHeaders(r.client.Config)) + var response CatalogRemote + err = apiClient.Do(ctx, method, path, headers, nil, request, &response) + if err != nil { + return nil, err + } + return &response, nil } func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { return &CatalogState{ CreateCatalog: input.CreateCatalog, IsolationMode: input.IsolationMode, + Id: "", // Recorded by the framework from the remote state (internal_id_field). } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { +func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { return &CatalogState{ Comment: info.Comment, ConnectionName: info.ConnectionName, @@ -58,16 +114,17 @@ func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), IsolationMode: info.IsolationMode, + Id: info.Id, } } -func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.CatalogInfo, error) { - return r.client.Catalogs.GetByName(ctx, id) +func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*CatalogRemote, error) { + return r.do(ctx, http.MethodGet, catalogsPath+"/"+id, nil) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) - if err != nil || response == nil { +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *CatalogRemote, error) { + response, err := r.do(ctx, http.MethodPost, catalogsPath, config.CreateCatalog) + if err != nil { return "", nil, err } if config.IsolationMode == "" { @@ -75,7 +132,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s } // UC only accepts isolation_mode on update, so a configured value takes a second call. - response, err = r.client.Catalogs.Update(ctx, catalog.UpdateCatalog{ + response, err = r.do(ctx, http.MethodPatch, catalogsPath+"/"+response.Name, catalog.UpdateCatalog{ Comment: "", CustomMaxRetentionHours: 0, EnablePredictiveOptimization: "", @@ -96,6 +153,11 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s return response.Name, response, nil } +// IDFromState returns the name, which is the ID. +func (*ResourceCatalog) IDFromState(state *CatalogState) string { + return state.Name +} + // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. // // CustomMaxRetentionHours is force-sent here even though schemaForceSend deliberately omits it: @@ -105,7 +167,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} // DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*CatalogRemote, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, @@ -120,16 +182,11 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *Catal ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } - response, err := r.client.Catalogs.Update(ctx, updateRequest) - if err != nil { - return nil, err - } - - return response, nil + return r.do(ctx, http.MethodPatch, catalogsPath+"/"+id, updateRequest) } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *CatalogRemote, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, @@ -148,7 +205,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config updateRequest.NewName = config.Name } - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.do(ctx, http.MethodPatch, catalogsPath+"/"+id, updateRequest) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/configs/catalogs.yml b/bundle/direct/dresources/configs/catalogs.yml index c1e1893af42..346d529795e 100644 --- a/bundle/direct/dresources/configs/catalogs.yml +++ b/bundle/direct/dresources/configs/catalogs.yml @@ -24,3 +24,6 @@ backend_defaults: # value alone (OPEN by default, or ISOLATED when set outside the bundle) rather than # forcing it back to OPEN. - field: isolation_mode +# Assigned by the backend and kept across renames: lets the plan find a catalog renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: id diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 53740b3bd89..7479dd0f8d7 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -6,9 +6,26 @@ import ( "net/http" "strings" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// fakeCatalog is CatalogInfo plus the id UC returns, which its API spec does not declare. The id +// is kept across renames. +type fakeCatalog struct { + catalog.CatalogInfo + + Id string `json:"id,omitempty"` +} + +func (c *fakeCatalog) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c fakeCatalog) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + // catalogNameManagedDefaults scopes the managed-property injection to the // backend-default drift test, keeping unrelated tests free of it. Mirrors // schemaNameManagedDefaults. @@ -85,9 +102,10 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } } - s.Catalogs[createRequest.Name] = catalogInfo + created := fakeCatalog{CatalogInfo: catalogInfo, Id: nextUUID()} + s.Catalogs[createRequest.Name] = created return Response{ - Body: catalogInfo, + Body: created, } } @@ -115,7 +133,7 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { } } - applyUpdatedFields(&existing, updateRequest, fields) + applyUpdatedFields(&existing.CatalogInfo, updateRequest, fields) if updateRequest.NewName != "" { existing.Name = updateRequest.NewName existing.FullName = updateRequest.NewName diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index de924c08060..ebd071dc9f0 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -38,7 +38,7 @@ func TestCatalogsCreate_AllowsNonEmptyName(t *testing.T) { getResponse := MapGet(workspace, workspace.Catalogs, "my_catalog") require.Equal(t, 0, getResponse.StatusCode) - body, ok := getResponse.Body.(catalog.CatalogInfo) + body, ok := getResponse.Body.(fakeCatalog) require.True(t, ok) assert.Equal(t, "my_catalog", body.Name) assert.Equal(t, "my_catalog", body.FullName) @@ -78,8 +78,9 @@ func TestCatalogsCreate_EchoesEveryAcceptedField(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) assert.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + body, ok := response.Body.(fakeCatalog) require.True(t, ok) + info := body.CatalogInfo assert.Equal(t, "my_catalog", info.Name) assert.Equal(t, "c", info.Comment) @@ -114,7 +115,7 @@ func TestCatalogsCreate_DefaultsIsolationModeToOpen(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) require.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + info, ok := response.Body.(fakeCatalog) require.True(t, ok) assert.Equal(t, catalog.CatalogIsolationModeOpen, info.IsolationMode) } @@ -136,8 +137,9 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { }`)}, "my_catalog") assert.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + body, ok := response.Body.(fakeCatalog) require.True(t, ok) + info := body.CatalogInfo assert.Equal(t, "updated", info.Comment) assert.Equal(t, int64(72), info.CustomMaxRetentionHours) @@ -147,3 +149,16 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { assert.Equal(t, map[string]string{"opt": "3"}, info.Options) assert.Equal(t, map[string]string{"prop": "4"}, info.Properties) } + +func TestCatalogsUpdate_RenameKeepsID(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + + created, ok := workspace.CatalogsCreate(Request{Body: []byte(`{"name": "my_catalog"}`)}).Body.(fakeCatalog) + require.True(t, ok) + require.NotEmpty(t, created.Id) + + renamed, ok := workspace.CatalogsUpdate(Request{Body: []byte(`{"new_name": "my_catalog2"}`)}, "my_catalog").Body.(fakeCatalog) + require.True(t, ok) + assert.Equal(t, "my_catalog2", renamed.Name) + assert.Equal(t, created.Id, renamed.Id) +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5e8068b6d02..338e4870a9c 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -242,7 +242,7 @@ type FakeWorkspace struct { ClusterLibraries map[string][]compute.Library // cluster id -> installed libraries InstancePools map[string]compute.GetInstancePool ClusterPolicies map[string]compute.Policy - Catalogs map[string]catalog.CatalogInfo + Catalogs map[string]fakeCatalog ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo ModelServices map[string]catalog.ModelService @@ -538,7 +538,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, - Catalogs: map[string]catalog.CatalogInfo{}, + Catalogs: map[string]fakeCatalog{}, ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{},