diff --git a/.nextchanges/bundles/parent-id-fields.md b/.nextchanges/bundles/parent-id-fields.md new file mode 100644 index 00000000000..054d31bf8e5 --- /dev/null +++ b/.nextchanges/bundles/parent-id-fields.md @@ -0,0 +1 @@ +* Delete the old resource when its name follows a renamed schema. ([#7718](https://github.com/databricks/cli/pull/7718)) diff --git a/.nextchanges/bundles/registered-model-rename.md b/.nextchanges/bundles/registered-model-rename.md new file mode 100644 index 00000000000..8b45402ed03 --- /dev/null +++ b/.nextchanges/bundles/registered-model-rename.md @@ -0,0 +1 @@ +* Rename registered models in place and follow parent renames. ([#7379](https://github.com/databricks/cli/pull/7379)) diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/output.txt b/acceptance/bundle/migrate/auto/registered-model-rename/output.txt index a81db3e5b61..3941779024e 100644 --- a/acceptance/bundle/migrate/auto/registered-model-rename/output.txt +++ b/acceptance/bundle/migrate/auto/registered-model-rename/output.txt @@ -3,19 +3,18 @@ >>> replay_tfstate.py terraform-requests.json terraform.tfstate -=== Rename the model (an immutable id field) and deploy on direct +=== Rename the model (an updatable id field) and deploy on direct >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: migration to direct: registered_models.my_model: name differs between config ("themodel-[UNIQUE_NAME]-renamed") and terraform state ("themodel-[UNIQUE_NAME]"); this resource will be recreated. +Warn: migration to direct: registered_models.my_model: name differs between config ("themodel-[UNIQUE_NAME]-renamed") and terraform state ("themodel-[UNIQUE_NAME]"); this resource will be renamed. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Migrated 2 resources to direct deployment engine. -Recreated registered_models.my_model +Updated registered_models.my_model Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> print_migration_telemetry -direct_migrate_recreate_planned true direct_migrated_via_env true === Redeploy on direct: the model now matches config, so the plan is a clean no-op diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/script b/acceptance/bundle/migrate/auto/registered-model-rename/script index 0b81939d6d4..94ffd9d6a25 100644 --- a/acceptance/bundle/migrate/auto/registered-model-rename/script +++ b/acceptance/bundle/migrate/auto/registered-model-rename/script @@ -10,12 +10,10 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate rm -f "$OUT_REQUESTS" -title "Rename the model (an immutable id field) and deploy on direct" -# name is a provided id field on registered models, so a rename recreates. reconcileIDFields -# seeds the deployed name into the converted state, so the migration's plan sees a real -# recreate. Unlike schemas/volumes, registered models are not in the destructive-approval -# group (a model recreate loses no data), so the deploy is approved without --auto-approve: -# the migration commits and the recreate applies on direct. +title "Rename the model (an updatable id field) and deploy on direct" +# name is an updatable id field: the model is renamed in place (UpdateWithID), keeping its +# versions. reconcileIDFields seeds the deployed name into the converted state, so the +# migration's plan sees a rename rather than snapshotting the new name as already applied. update_file.py databricks.yml "themodel-$UNIQUE_NAME" "themodel-$UNIQUE_NAME-renamed" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry diff --git a/acceptance/bundle/migrate/auto/volume-type-change/script b/acceptance/bundle/migrate/auto/volume-type-change/script index ea5ebdb1499..922c77c7b0c 100644 --- a/acceptance/bundle/migrate/auto/volume-type-change/script +++ b/acceptance/bundle/migrate/auto/volume-type-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: changing volume_type to EXTERNAL needs a storage_location backed by a # registered external location on a real workspace, so this is not Cloud-enabled. The -# recreate class it covers is verified on cloud by schema-catalog-change / registered-model-rename. +# recreate class it covers is verified on cloud by schema-catalog-change / registered-model-catalog-change. # volume_type is both a provided id field and a recreate_on_changes field. reconcileIDFields # records the deployed value, so the migration's plan sees a real change and recreates. Volumes diff --git a/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl index e45eccddc44..37a21cc13da 100644 --- a/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl +++ b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl @@ -14,3 +14,8 @@ resources: catalog_name: ${resources.catalogs.catalog1.name} schema_name: ${resources.schemas.schema1.name} name: myvolume + registered_models: + model1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mymodel diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 0d5b9fcb881..5bb6c4b2655 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -1,15 +1,16 @@ -=== Deploy a catalog with a schema and a volume in it +=== Deploy a catalog with a schema, a volume and a registered model in it >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 +Created registered_models.model1 Created schemas.schema1 Created volumes.volume1 Files: 2 uploaded, 0 deleted -Resources: 3 created, 0 changed, 0 deleted, 0 unchanged +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog { "method": "POST", "path": "/api/2.1/unity-catalog/catalogs", @@ -17,6 +18,15 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "name": "test_catalog_[UNIQUE_NAME]" } } +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mymodel", + "schema_name": "myschema" + } +} { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", @@ -40,7 +50,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -=== UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them +=== UC moves the schema, the volume and the model with the catalog, so the plan re-keys them instead of recreating them >>> [CLI] bundle plan --output json { @@ -53,7 +63,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", "name": "test_catalog_[UNIQUE_NAME]", @@ -69,6 +78,59 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } }, + "resources.registered_models.model1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "mymodel", + "schema_name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mymodel", + "name": "mymodel", + "owner": "[USERNAME]", + "schema_name": "myschema" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "created_by": { + "action": "skip", + "reason": "output_only", + "remote": "[USERNAME]" + }, + "full_name": { + "action": "skip", + "reason": "backend_default", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mymodel" + }, + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + } + } + }, "resources.schemas.schema1": { "depends_on": [ { @@ -87,7 +149,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", @@ -133,7 +194,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume", "name": "myvolume", @@ -160,13 +220,13 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request +=== Deploy renames the catalog. UC moves the schema, the volume and the model with it, so they get no request >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged >>> print_requests.py //unity-catalog { @@ -179,7 +239,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } } -=== The old catalog is gone and the schema and the volume are under the new one, with their original ids +=== The old catalog is gone and the schema, the volume and the model are under the new one, not recreated old catalog is gone >>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema @@ -196,6 +256,13 @@ old catalog is gone "volume_id": "[VOLUME_ID]" } +>>> [CLI] registered-models get test_catalog_renamed_[UNIQUE_NAME].myschema.mymodel +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.mymodel", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [MODEL_CREATED_AT] +} + === The state tracks the new paths >>> [CLI] bundle summary --output json @@ -203,6 +270,9 @@ old catalog is gone "catalogs": { "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" }, + "registered_models": { + "model1": "test_catalog_renamed_[UNIQUE_NAME].myschema.mymodel" + }, "schemas": { "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" }, @@ -214,18 +284,19 @@ old catalog is gone === Redeploy is a no-op >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 3 unchanged +Resources: 0 created, 0 changed, 0 deleted, 4 unchanged >>> print_requests.py //unity-catalog >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.catalog1 + delete resources.registered_models.model1 delete resources.schemas.schema1 delete resources.volumes.volume1 @@ -240,4 +311,4 @@ is removed from the catalog, but the underlying files are not deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 3 deleted +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 1b20c07091b..30c98f549ca 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -11,28 +11,31 @@ cleanup() { } trap cleanup EXIT -title "Deploy a catalog with a schema and a volume in it" +title "Deploy a catalog with a schema, a volume and a registered model in it" trace $CLI bundle deploy -trace print_requests.py //unity-catalog +# The volume and the model are created in parallel. +trace print_requests.py --sort //unity-catalog add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID +add_repl "$($CLI registered-models get "$OLD_CATALOG.myschema.mymodel" | jq -r .created_at)" MODEL_CREATED_AT title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" +title "UC moves the schema, the volume and the model with the catalog, so the plan re-keys them instead of recreating them" # Drop the remote_state fields that differ between clouds or from the test server. -trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location -title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" +title "Deploy renames the catalog. UC moves the schema, the volume and the model with it, so they get no request" trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog -title "The old catalog is gone and the schema and the volume are under the new one, with their original ids" +title "The old catalog is gone and the schema, the volume and the model are under the new one, not recreated" # Not traced: the "not found" message differs between UC and the test server. $CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name, schema_id}' trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name, volume_id}' +trace $CLI registered-models get "$NEW_CATALOG.myschema.mymodel" | jq '{full_name, catalog_name, created_at}' title "The state tracks the new paths" trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/grants/registered_model_change_name/databricks.yml.tmpl b/acceptance/bundle/resources/grants/registered_model_change_name/databricks.yml.tmpl new file mode 100644 index 00000000000..f3fbb05ae6d --- /dev/null +++ b/acceptance/bundle/resources/grants/registered_model_change_name/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel + grants: + - principal: deco-test-user@databricks.com + privileges: + - APPLY_TAG diff --git a/acceptance/bundle/resources/grants/registered_model_change_name/out.test.toml b/acceptance/bundle/resources/grants/registered_model_change_name/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/grants/registered_model_change_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/grants/registered_model_change_name/output.txt b/acceptance/bundle/resources/grants/registered_model_change_name/output.txt new file mode 100644 index 00000000000..a53f0e03a28 --- /dev/null +++ b/acceptance/bundle/resources/grants/registered_model_change_name/output.txt @@ -0,0 +1,130 @@ + +=== Deploy a registered model with grants + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created registered_models.model1.grants +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "mymodel", + "schema_name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/function/main.test-schema-[UNIQUE_NAME].mymodel", + "body": { + "changes": [ + { + "add": [ + "APPLY_TAG" + ], + "principal": "deco-test-user@databricks.com", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} + +>>> read_state.py registered_models model1.grants id +registered_models model1.grants id='function/main.test-schema-[UNIQUE_NAME].mymodel' + +=== Rename the model: its ID changes, and so does the ID of its grants + +>>> update_file.py databricks.yml mymodel mymodel_renamed + +>>> [CLI] bundle plan --output json +{ + "full_name": { + "action": "update_id", + "reason": "id_changes", + "old": "main.test-schema-[UNIQUE_NAME].mymodel", + "new": "", + "remote": "main.test-schema-[UNIQUE_NAME].mymodel" + } +} + +=== Deploy renames the model in place and applies the grants under the new name + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 +Updated registered_models.model1.grants +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema-[UNIQUE_NAME].mymodel", + "body": { + "catalog_name": "main", + "comment": "", + "new_name": "mymodel_renamed", + "schema_name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/function/main.test-schema-[UNIQUE_NAME].mymodel_renamed", + "body": { + "changes": [ + { + "add": [ + "APPLY_TAG" + ], + "principal": "deco-test-user@databricks.com", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} + +>>> [CLI] grants get function main.test-schema-[UNIQUE_NAME].mymodel_renamed +{ + "principal": "deco-test-user@databricks.com", + "privileges": [ + "APPLY_TAG" + ] +} + +=== The grants are recorded under the new name, so the next plan is a no-op + +>>> read_state.py registered_models model1.grants id +registered_models model1.grants id='function/main.test-schema-[UNIQUE_NAME].mymodel_renamed' + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/grants/registered_model_change_name/script b/acceptance/bundle/resources/grants/registered_model_change_name/script new file mode 100644 index 00000000000..c2668c183e5 --- /dev/null +++ b/acceptance/bundle/resources/grants/registered_model_change_name/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_MODEL="main.test-schema-${UNIQUE_NAME}.mymodel_renamed" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a registered model with grants" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace read_state.py registered_models model1.grants id + +title "Rename the model: its ID changes, and so does the ID of its grants" +trace update_file.py databricks.yml mymodel mymodel_renamed +trace $CLI bundle plan --output json | jq '.plan["resources.registered_models.model1.grants"].changes' + +title "Deploy renames the model in place and applies the grants under the new name" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI grants get function "$NEW_MODEL" | jq '.privilege_assignments[] | select(.principal == "deco-test-user@databricks.com")' + +title "The grants are recorded under the new name, so the next plan is a no-op" +trace read_state.py registered_models model1.grants id +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/registered_models/basic/output.txt b/acceptance/bundle/resources/registered_models/basic/output.txt index fb05660caf4..9ee65363440 100644 --- a/acceptance/bundle/resources/registered_models/basic/output.txt +++ b/acceptance/bundle/resources/registered_models/basic/output.txt @@ -61,18 +61,18 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "schema_name": "default" } -=== update the name, this should recreate +=== update the name, this renames the model in place >>> [CLI] bundle plan -recreate registered_models.my_registered_model +update registered_models.my_registered_model -Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-registered-models-basic-[UNIQUE_NAME]/default/files... -Recreated registered_models.my_registered_model +Updated registered_models.my_registered_model Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 0 unchanged +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] registered-models get main.default.my-registered-model-updated-[UNIQUE_NAME] { diff --git a/acceptance/bundle/resources/registered_models/basic/script b/acceptance/bundle/resources/registered_models/basic/script index 61dff74182c..91b57c1e80a 100644 --- a/acceptance/bundle/resources/registered_models/basic/script +++ b/acceptance/bundle/resources/registered_models/basic/script @@ -36,7 +36,7 @@ deploy_registered_model export NAME="my-registered-model-updated-$UNIQUE_NAME" envsubst < databricks.yml.tmpl > databricks.yml -title "update the name, this should recreate" +title "update the name, this renames the model in place" deploy_registered_model title "update the catalog name, this should recreate" diff --git a/acceptance/bundle/resources/registered_models/move_from_renamed_schema/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..d5d45b984fe --- /dev/null +++ b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema1-$UNIQUE_NAME + schema2: + catalog_name: main + name: test-schema2-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel diff --git a/acceptance/bundle/resources/registered_models/move_from_renamed_schema/out.test.toml b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/out.test.toml new file mode 100644 index 00000000000..7a62f726f25 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/registered_models/move_from_renamed_schema/output.txt b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/output.txt new file mode 100644 index 00000000000..b3fafcb0a62 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/output.txt @@ -0,0 +1,190 @@ + +=== Deploy two schemas and a registered model in the first one + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.schema1 +Created schemas.schema2 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the first schema and move the model to the second one in one deploy. Moving the model recreates it + +>>> update_file.py databricks.yml test-schema1- test-schema1-renamed- + +>>> update_file.py databricks.yml schemas.schema1.name schemas.schema2.name + +>>> [CLI] bundle plan --output json +{ + "resources.registered_models.model1": { + "id": "main.test-schema1-renamed-[UNIQUE_NAME].mymodel", + "depends_on": [ + { + "node": "resources.schemas.schema2", + "label": "${resources.schemas.schema2.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "catalog_name": "main", + "name": "mymodel", + "schema_name": "test-schema2-[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_name": "main", + "created_by": "[USERNAME]", + "full_name": "main.test-schema1-[UNIQUE_NAME].mymodel", + "name": "mymodel", + "owner": "[USERNAME]", + "schema_name": "test-schema1-[UNIQUE_NAME]" + }, + "changes": { + "created_by": { + "action": "skip", + "reason": "output_only", + "remote": "[USERNAME]" + }, + "full_name": { + "action": "skip", + "reason": "backend_default", + "remote": "main.test-schema1-[UNIQUE_NAME].mymodel" + }, + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema1-[UNIQUE_NAME]", + "new": "test-schema2-[UNIQUE_NAME]", + "remote": "test-schema1-[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema1-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema1-[UNIQUE_NAME]", + "name": "test-schema1-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID][0]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema1-[UNIQUE_NAME]", + "new": "test-schema1-renamed-[UNIQUE_NAME]", + "remote": "test-schema1-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } + } + }, + "resources.schemas.schema2": { + "action": "skip", + "remote_state": { + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema2-[UNIQUE_NAME]", + "name": "test-schema2-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID][1]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][1]", + "remote": "[UUID][1]" + } + } + } +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated registered_models.model1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/models/main.test-schema1-renamed-[UNIQUE_NAME].mymodel" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "mymodel", + "schema_name": "test-schema2-[UNIQUE_NAME]" + } +} + +=== The model is in the second schema + +>>> [CLI] registered-models get main.test-schema2-[UNIQUE_NAME].mymodel +{ + "full_name": "main.test-schema2-[UNIQUE_NAME].mymodel" +} + +=== The old model, moved with the renamed schema, was deleted there + +>>> musterr [CLI] registered-models get main.test-schema1-renamed-[UNIQUE_NAME].mymodel +Error: Routine or Model 'main.test-schema1-renamed-[UNIQUE_NAME].mymodel' does not exist. + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.schema1 + delete resources.schemas.schema2 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/registered_models/move_from_renamed_schema/script b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/script new file mode 100644 index 00000000000..5c065d6da5d --- /dev/null +++ b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/script @@ -0,0 +1,33 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +RENAMED_MODEL="main.test-schema1-renamed-${UNIQUE_NAME}.mymodel" +NEW_MODEL="main.test-schema2-${UNIQUE_NAME}.mymodel" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy two schemas and a registered model in the first one" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the first schema and move the model to the second one in one deploy. Moving the model recreates it" +trace update_file.py databricks.yml test-schema1- test-schema1-renamed- +trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag storage_location +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) --auto-approve +trace print_requests.py //unity-catalog + +title "The model is in the second schema" +trace $CLI registered-models get "$NEW_MODEL" | jq '{full_name}' + +title "The old model, moved with the renamed schema, was deleted there" +trace musterr $CLI registered-models get "$RENAMED_MODEL" + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/registered_models/move_from_renamed_schema/test.toml b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/test.toml new file mode 100644 index 00000000000..ebd892fd9df --- /dev/null +++ b/acceptance/bundle/resources/registered_models/move_from_renamed_schema/test.toml @@ -0,0 +1,14 @@ +Cloud = true +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/registered_models/parent_rename_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..6f739c9c6c5 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel + comment: c1 diff --git a/acceptance/bundle/resources/registered_models/parent_rename_interrupted/out.test.toml b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/registered_models/parent_rename_interrupted/output.txt b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/output.txt new file mode 100644 index 00000000000..a8d0a2cb96a --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/output.txt @@ -0,0 +1,184 @@ + +=== Deploy a schema with a registered model in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change the model's comment. The CLI is killed on the model's update, after the schema's rename is saved + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> errcode [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The model's new path was recorded before its update, so the plan updates it there + +>>> [CLI] bundle plan --output json +{ + "resources.registered_models.model1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c2", + "name": "mymodel", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_name": "main", + "comment": "c1", + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mymodel", + "metastore_id": "[UUID][0]", + "name": "mymodel", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "updated_by": "[USERNAME]" + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "created_by": { + "action": "skip", + "reason": "output_only", + "remote": "[USERNAME]" + }, + "full_name": { + "action": "skip", + "reason": "backend_default", + "remote": "main.test-schema-renamed-[UNIQUE_NAME].mymodel" + }, + "metastore_id": { + "action": "skip", + "reason": "backend_default", + "remote": "[UUID][0]" + }, + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + }, + "updated_by": { + "action": "skip", + "reason": "output_only", + "remote": "[USERNAME]" + } + } + }, + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "metastore_id": "[UUID][1]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID][2]", + "updated_by": "[USERNAME]" + }, + "changes": { + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][2]", + "remote": "[UUID][2]" + } + } + } +} + +=== The next deploy updates the model at its new path + +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema-renamed-[UNIQUE_NAME].mymodel", + "body": { + "catalog_name": "main", + "comment": "c2", + "name": "mymodel", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +>>> [CLI] registered-models get main.test-schema-renamed-[UNIQUE_NAME].mymodel +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mymodel", + "comment": "c2", + "created_at": [MODEL_CREATED_AT] +} + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/registered_models/parent_rename_interrupted/script b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/script new file mode 100644 index 00000000000..4423e4b28cd --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_MODEL="main.test-schema-renamed-${UNIQUE_NAME}.mymodel" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a registered model in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI registered-models get "main.test-schema-${UNIQUE_NAME}.mymodel" | jq -r .created_at)" MODEL_CREATED_AT + +title "Rename the schema and change the model's comment. The CLI is killed on the model's update, after the schema's rename is saved" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "comment: c1" "comment: c2" +kill_after.py "PATCH /api/2.1/unity-catalog/models/$NEW_MODEL" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q +trace print_requests.py //unity-catalog + +title "The model's new path was recorded before its update, so the plan updates it there" +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at + +title "The next deploy updates the model at its new path" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI registered-models get "$NEW_MODEL" | jq '{full_name, comment, created_at}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/registered_models/parent_rename_interrupted/test.toml b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/test.toml new file mode 100644 index 00000000000..443e541f555 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_interrupted/test.toml @@ -0,0 +1,35 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace cannot. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# "Uploading bundle files" line, so it is followed directly by "Exit code: 1". +[[Repls]] +Old = '(Uploading bundle files to [^\n]+)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" diff --git a/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..630cfc75a4a --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel diff --git a/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/out.test.toml b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/output.txt b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/output.txt new file mode 100644 index 00000000000..7412e7de1a0 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/output.txt @@ -0,0 +1,67 @@ + +=== Deploy a schema with a registered model in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band. UC moves the model with it + +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]" +} + +=== With the config renamed to match, the plan finds the schema by its schema_id. A model has no such id, so the plan creates it again + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan +create registered_models.model1 + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== The create fails: the model is already there + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.registered_models.model1: Routine or Model 'mymodel' already exists (400 RESOURCE_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/models +HTTP Status: 400 Bad Request +API error_code: RESOURCE_ALREADY_EXISTS +API message: Routine or Model 'mymodel' already exists + +Files: 3 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "mymodel", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +>>> [CLI] registered-models get main.test-schema-renamed-[UNIQUE_NAME].mymodel +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mymodel" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/script b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/script new file mode 100644 index 00000000000..bbd254c0797 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a registered model in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the schema out of band. UC moves the model with it" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config renamed to match, the plan finds the schema by its schema_id. A model has no such id, so the plan creates it again" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace $CLI bundle plan + +title "The create fails: the model is already there" +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI registered-models get "$NEW_SCHEMA.mymodel" | jq '{full_name}' diff --git a/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/test.toml b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/test.toml new file mode 100644 index 00000000000..e1372ecbad3 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/parent_rename_out_of_band/test.toml @@ -0,0 +1,8 @@ +Cloud = true +Badness = "The plan does not look for a model whose schema was renamed outside the bundle (a model has no recorded id to find it by), so it creates it again: the create fails with RESOURCE_ALREADY_EXISTS on every deploy. Same on main." + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/registered_models/rename/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..6f739c9c6c5 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel + comment: c1 diff --git a/acceptance/bundle/resources/registered_models/rename/out.test.toml b/acceptance/bundle/resources/registered_models/rename/out.test.toml new file mode 100644 index 00000000000..7a62f726f25 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/registered_models/rename/output.txt b/acceptance/bundle/resources/registered_models/rename/output.txt new file mode 100644 index 00000000000..d26ba832ca5 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename/output.txt @@ -0,0 +1,190 @@ + +=== Deploy a schema with a registered model in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "comment": "c1", + "name": "mymodel", + "schema_name": "test-schema-[UNIQUE_NAME]" + } +} + +=== Rename the model: it is renamed in place, not recreated + +>>> update_file.py databricks.yml name: mymodel name: mymodel2 + +>>> [CLI] bundle plan --output json +{ + "resources.registered_models.model1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c1", + "name": "mymodel2", + "schema_name": "test-schema-[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_name": "main", + "comment": "c1", + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].mymodel", + "name": "mymodel", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]" + }, + "changes": { + "created_by": { + "action": "skip", + "reason": "output_only", + "remote": "[USERNAME]" + }, + "full_name": { + "action": "skip", + "reason": "backend_default", + "remote": "main.test-schema-[UNIQUE_NAME].mymodel" + }, + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "mymodel", + "new": "mymodel2", + "remote": "mymodel" + }, + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + } + } + }, + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema-[UNIQUE_NAME].mymodel", + "body": { + "catalog_name": "main", + "comment": "c1", + "new_name": "mymodel2", + "schema_name": "test-schema-[UNIQUE_NAME]" + } +} + +>>> [CLI] registered-models get main.test-schema-[UNIQUE_NAME].mymodel2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].mymodel2", + "created_at": [MODEL_CREATED_AT] +} + +=== Rename the model and change its comment in one deploy: both go in one PATCH + +>>> update_file.py databricks.yml name: mymodel2 name: mymodel3 + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema-[UNIQUE_NAME].mymodel2", + "body": { + "catalog_name": "main", + "comment": "c2", + "new_name": "mymodel3", + "schema_name": "test-schema-[UNIQUE_NAME]" + } +} + +>>> [CLI] registered-models get main.test-schema-[UNIQUE_NAME].mymodel3 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].mymodel3", + "comment": "c2", + "created_at": [MODEL_CREATED_AT] +} + +=== The state tracks the new name and the plan is a no-op + +>>> [CLI] bundle summary --output json +{ + "registered_models": { + "model1": "main.test-schema-[UNIQUE_NAME].mymodel3" + }, + "schemas": { + "schema1": "main.test-schema-[UNIQUE_NAME]" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/registered_models/rename/script b/acceptance/bundle/resources/registered_models/rename/script new file mode 100644 index 00000000000..4945c5ccd92 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a registered model in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI registered-models get "$SCHEMA.mymodel" | jq -r .created_at)" MODEL_CREATED_AT + +title "Rename the model: it is renamed in place, not recreated" +trace update_file.py databricks.yml "name: mymodel" "name: mymodel2" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag storage_location +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) +trace print_requests.py //unity-catalog +trace $CLI registered-models get "$SCHEMA.mymodel2" | jq '{full_name, created_at}' + +title "Rename the model and change its comment in one deploy: both go in one PATCH" +trace update_file.py databricks.yml "name: mymodel2" "name: mymodel3" +trace update_file.py databricks.yml "comment: c1" "comment: c2" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI registered-models get "$SCHEMA.mymodel3" | jq '{full_name, comment, created_at}' + +title "The state tracks the new name and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/registered_models/rename/test.toml b/acceptance/bundle/resources/registered_models/rename/test.toml new file mode 100644 index 00000000000..ebd892fd9df --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename/test.toml @@ -0,0 +1,14 @@ +Cloud = true +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..957167b6ad5 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + registered_models: + model1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mymodel diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/out.test.toml b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/out.test.toml new file mode 100644 index 00000000000..7a62f726f25 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt new file mode 100644 index 00000000000..895e3dce8a8 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt @@ -0,0 +1,220 @@ + +=== Deploy a catalog with a schema and a registered model in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created registered_models.model1 +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog and the schema in one deploy + +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +=== Both of the model's parents are renamed, so both its catalog_name and schema_name follow them + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.registered_models.model1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "mymodel", + "schema_name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mymodel", + "name": "mymodel", + "owner": "[USERNAME]", + "schema_name": "myschema" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "created_by": { + "action": "skip", + "reason": "output_only", + "remote": "[USERNAME]" + }, + "full_name": { + "action": "skip", + "reason": "backend_default", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mymodel" + }, + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + } +} + +=== Deploy renames the catalog, then the schema under it. UC moves the model both times, so it gets no request + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +=== The model is under the new names and was not recreated + +>>> [CLI] registered-models get test_catalog_renamed_[UNIQUE_NAME].myschema2.mymodel +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mymodel", + "created_at": [MODEL_CREATED_AT] +} + +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "registered_models": { + "model1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mymodel" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + } +} + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.registered_models.model1 + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/script b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/script new file mode 100644 index 00000000000..676a9b261c8 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_SCHEMA="test_catalog_renamed_${UNIQUE_NAME}.myschema2" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a schema and a registered model in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI registered-models get "test_catalog_${UNIQUE_NAME}.myschema.mymodel" | jq -r .created_at)" MODEL_CREATED_AT + +title "Rename the catalog and the schema in one deploy" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "name: myschema" "name: myschema2" + +title "Both of the model's parents are renamed, so both its catalog_name and schema_name follow them" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + +title "Deploy renames the catalog, then the schema under it. UC moves the model both times, so it gets no request" +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) +trace print_requests.py //unity-catalog + +title "The model is under the new names and was not recreated" +trace $CLI registered-models get "$NEW_SCHEMA.mymodel" | jq '{full_name, created_at}' +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/test.toml b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/test.toml new file mode 100644 index 00000000000..ebd892fd9df --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/test.toml @@ -0,0 +1,14 @@ +Cloud = true +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/registered_models/rename_to_existing_name/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/rename_to_existing_name/databricks.yml.tmpl new file mode 100644 index 00000000000..90d305eff4d --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_to_existing_name/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: model_a + model2: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: model_b diff --git a/acceptance/bundle/resources/registered_models/rename_to_existing_name/out.test.toml b/acceptance/bundle/resources/registered_models/rename_to_existing_name/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_to_existing_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/registered_models/rename_to_existing_name/output.txt b/acceptance/bundle/resources/registered_models/rename_to_existing_name/output.txt new file mode 100644 index 00000000000..ec8a113e5a2 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_to_existing_name/output.txt @@ -0,0 +1,62 @@ + +=== Deploy a schema with two registered models + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created registered_models.model2 +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the first model to the second one's name: UC rejects the rename + +>>> update_file.py databricks.yml name: model_a name: model_b + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot update_id resources.registered_models.model1: updating id=main.test-schema-[UNIQUE_NAME].model_a: Routine or Model 'model_b' already exists (400 RESOURCE_ALREADY_EXISTS) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/models/main.test-schema-[UNIQUE_NAME].model_a +HTTP Status: 400 Bad Request +API error_code: RESOURCE_ALREADY_EXISTS +API message: Routine or Model 'model_b' already exists + +Files: 2 uploaded, 0 deleted + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema-[UNIQUE_NAME].model_a", + "body": { + "catalog_name": "main", + "comment": "", + "new_name": "model_b", + "schema_name": "test-schema-[UNIQUE_NAME]" + } +} + +=== Both models are still there under their names + +>>> [CLI] registered-models get main.test-schema-[UNIQUE_NAME].model_a +{ + "full_name": "main.test-schema-[UNIQUE_NAME].model_a" +} + +>>> [CLI] registered-models get main.test-schema-[UNIQUE_NAME].model_b +{ + "full_name": "main.test-schema-[UNIQUE_NAME].model_b" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.registered_models.model2 + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/registered_models/rename_to_existing_name/script b/acceptance/bundle/resources/registered_models/rename_to_existing_name/script new file mode 100644 index 00000000000..404e86e9aa9 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_to_existing_name/script @@ -0,0 +1,24 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with two registered models" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the first model to the second one's name: UC rejects the rename" +trace update_file.py databricks.yml "name: model_a" "name: model_b" +trace musterr $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Both models are still there under their names" +trace $CLI registered-models get "$SCHEMA.model_a" | jq '{full_name}' +trace $CLI registered-models get "$SCHEMA.model_b" | jq '{full_name}' diff --git a/acceptance/bundle/resources/registered_models/rename_to_existing_name/test.toml b/acceptance/bundle/resources/registered_models/rename_to_existing_name/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_to_existing_name/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/registered_models/rename_with_aliases/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/rename_with_aliases/databricks.yml.tmpl new file mode 100644 index 00000000000..e35701abb3a --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_with_aliases/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + registered_models: + model1: + catalog_name: main + schema_name: default + name: model-$UNIQUE_NAME + aliases: [] diff --git a/acceptance/bundle/resources/registered_models/rename_with_aliases/out.test.toml b/acceptance/bundle/resources/registered_models/rename_with_aliases/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_with_aliases/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/registered_models/rename_with_aliases/output.txt b/acceptance/bundle/resources/registered_models/rename_with_aliases/output.txt new file mode 100644 index 00000000000..b6089a9e821 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_with_aliases/output.txt @@ -0,0 +1,139 @@ + +=== Deploy a model and register two versions of it out of band + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> register_model_version main.default.model-[UNIQUE_NAME] +1 + +>>> register_model_version main.default.model-[UNIQUE_NAME] +2 + +=== Rename the model and point 'champion' at version 2 in one deploy. The alias is set first, under the current name + +>>> update_file.py databricks.yml name: model- name: model2- + +>>> update_file.py databricks.yml aliases: [] aliases: [{alias_name: champion, version_num: 2}] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PUT", + "path": "/api/2.1/unity-catalog/models/main.default.model-[UNIQUE_NAME]/aliases/champion", + "body": { + "version_num": 2 + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.default.model-[UNIQUE_NAME]", + "body": { + "catalog_name": "main", + "comment": "", + "new_name": "model2-[UNIQUE_NAME]", + "schema_name": "default" + } +} + +>>> [CLI] registered-models get main.default.model2-[UNIQUE_NAME] --include-aliases +[ + { + "alias_name": "champion", + "version_num": 2 + } +] + +=== The versions moved with the model: the next one registered is version 3 + +>>> register_model_version main.default.model2-[UNIQUE_NAME] +3 + +=== Rename it again and point 'champion' at version 9, which does not exist. The alias fails first, so the model keeps its name + +>>> update_file.py databricks.yml name: model2- name: model3- + +>>> update_file.py databricks.yml version_num: 2 version_num: 9 + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot update_id resources.registered_models.model1: updating id=main.default.model2-[UNIQUE_NAME]: Model version 9 of registered model main.default.model2-[UNIQUE_NAME] does not exist. (404 RESOURCE_DOES_NOT_EXIST) + +Endpoint: PUT [DATABRICKS_URL]/api/2.1/unity-catalog/models/main.default.model2-[UNIQUE_NAME]/aliases/champion +HTTP Status: 404 Not Found +API error_code: RESOURCE_DOES_NOT_EXIST +API message: Model version 9 of registered model main.default.model2-[UNIQUE_NAME] does not exist. + +Files: 2 uploaded, 0 deleted + +>>> print_requests.py //unity-catalog +{ + "method": "PUT", + "path": "/api/2.1/unity-catalog/models/main.default.model2-[UNIQUE_NAME]/aliases/champion", + "body": { + "version_num": 9 + } +} + +>>> [CLI] registered-models get main.default.model2-[UNIQUE_NAME] +{ + "full_name": "main.default.model2-[UNIQUE_NAME]" +} + +=== With the alias fixed, the next deploy renames the model + +>>> update_file.py databricks.yml version_num: 9 version_num: 1 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PUT", + "path": "/api/2.1/unity-catalog/models/main.default.model2-[UNIQUE_NAME]/aliases/champion", + "body": { + "version_num": 1 + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.default.model2-[UNIQUE_NAME]", + "body": { + "catalog_name": "main", + "comment": "", + "new_name": "model3-[UNIQUE_NAME]", + "schema_name": "default" + } +} + +>>> [CLI] registered-models get main.default.model3-[UNIQUE_NAME] --include-aliases +[ + { + "alias_name": "champion", + "version_num": 1 + } +] + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/registered_models/rename_with_aliases/script b/acceptance/bundle/resources/registered_models/rename_with_aliases/script new file mode 100644 index 00000000000..1b3ea62fa66 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_with_aliases/script @@ -0,0 +1,42 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a model and register two versions of it out of band" +trace $CLI bundle deploy +trace register_model_version "main.default.model-$UNIQUE_NAME" +trace register_model_version "main.default.model-$UNIQUE_NAME" +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the model and point 'champion' at version 2 in one deploy. The alias is set first, under the current name" +trace update_file.py databricks.yml "name: model-" "name: model2-" +trace update_file.py databricks.yml "aliases: []" "aliases: [{alias_name: champion, version_num: 2}]" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI registered-models get "main.default.model2-$UNIQUE_NAME" --include-aliases | alias_fields + +title "The versions moved with the model: the next one registered is version 3" +trace register_model_version "main.default.model2-$UNIQUE_NAME" +print_requests.py //unity-catalog &> LOG.register_requests + +title "Rename it again and point 'champion' at version 9, which does not exist. The alias fails first, so the model keeps its name" +trace update_file.py databricks.yml "name: model2-" "name: model3-" +trace update_file.py databricks.yml "version_num: 2" "version_num: 9" +trace musterr $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI registered-models get "main.default.model2-$UNIQUE_NAME" | jq '{full_name}' + +title "With the alias fixed, the next deploy renames the model" +trace update_file.py databricks.yml "version_num: 9" "version_num: 1" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI registered-models get "main.default.model3-$UNIQUE_NAME" --include-aliases | alias_fields + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/registered_models/rename_with_aliases/test.toml b/acceptance/bundle/resources/registered_models/rename_with_aliases/test.toml new file mode 100644 index 00000000000..ba7a8832919 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/rename_with_aliases/test.toml @@ -0,0 +1,9 @@ +# Local only: aliases need registered model versions, which the test registers through the fake +# server rather than with MLflow. +RecordRequests = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index edf4f618ffb..725161c1e8f 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -31,13 +31,7 @@ Exit code: [KILLED] } } -=== The volume is gone from its recorded path, so the plan looks for it at its path in config - ->>> print_state.py -{ - "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "volume_id": "[VOLUME_ID]" -} +=== The volume's new path was recorded before its update, so the plan updates it there >>> [CLI] bundle plan --output json { @@ -83,14 +77,13 @@ Exit code: [KILLED] } }, "resources.volumes.volume1": { - "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", "depends_on": [ { "node": "resources.schemas.schema1", "label": "${resources.schemas.schema1.name}" } ], - "action": "update_id", + "action": "update", "new_state": { "value": { "catalog_name": "main", @@ -119,13 +112,6 @@ Exit code: [KILLED] "new": "c2", "remote": "c1" }, - "schema_name": { - "action": "skip", - "reason": "remote_already_set", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-renamed-[UNIQUE_NAME]" - }, "storage_location": { "action": "skip", "reason": "immutable_backend_value", diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script index c829d0dea23..8796dc3e26e 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/script +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -23,8 +23,7 @@ kill_after.py "PATCH /api/2.1/unity-catalog/volumes/$NEW_VOLUME" 0 1 trace errcode $CLI bundle deploy -q trace print_requests.py //unity-catalog -title "The volume is gone from its recorded path, so the plan looks for it at its path in config" -trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' +title "The volume's new path was recorded before its update, so the plan updates it there" trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at title "The next deploy updates the volume at its new path" diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_referenced_by_name/databricks.yml.tmpl new file mode 100644 index 00000000000..730c34678c5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/databricks.yml.tmpl @@ -0,0 +1,27 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + schemas: + app: + catalog_name: main + name: test_app_$UNIQUE_NAME + shared: + catalog_name: main + name: test_shared_$UNIQUE_NAME + # Both live in the shared schema and take their own name from the app schema's name. + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.shared.name} + name: ${resources.schemas.app.name} + secrets: + secret1: + catalog_name: main + schema_name: ${resources.schemas.shared.name} + name: ${resources.schemas.app.name} + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/out.test.toml b/acceptance/bundle/resources/schemas/rename_referenced_by_name/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/output.txt b/acceptance/bundle/resources/schemas/rename_referenced_by_name/output.txt new file mode 100644 index 00000000000..eb0b3f7f3e7 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/output.txt @@ -0,0 +1,99 @@ + +=== Deploy two schemas, and a model and a secret in the shared one, named after the app schema + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.app +Created schemas.shared +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the app schema. The model and the secret stay in the shared schema, so only their own name changes + +>>> update_file.py databricks.yml test_app_ test_app_renamed_ + +>>> [CLI] bundle plan +update registered_models.model1 +update schemas.app +recreate secrets.secret1 + +Plan: 1 to add, 2 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated secrets.secret1 +Updated registered_models.model1 +Updated schemas.app +Files: 2 uploaded, 0 deleted +Resources: 1 created, 2 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --sort //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/secrets/main.test_shared_[UNIQUE_NAME].test_app_[UNIQUE_NAME]" +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test_shared_[UNIQUE_NAME].test_app_[UNIQUE_NAME]", + "body": { + "catalog_name": "main", + "comment": "", + "new_name": "test_app_renamed_[UNIQUE_NAME]", + "schema_name": "test_shared_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test_app_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_app_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "name": "test_app_renamed_[UNIQUE_NAME]", + "schema_name": "test_shared_[UNIQUE_NAME]", + "value": "secret-value" + } +} + +=== The old model and secret are gone; the new ones are in the shared schema +old model is gone +old secret is gone + +>>> [CLI] registered-models get main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME] +{ + "full_name": "main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME]" +} + +>>> [CLI] secrets-uc get-secret main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME] +{ + "full_name": "main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME]" +} + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.app + delete resources.schemas.shared + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.app + delete resources.schemas.shared + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/script b/acceptance/bundle/resources/schemas/rename_referenced_by_name/script new file mode 100644 index 00000000000..7cb6a142926 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +SHARED="main.test_shared_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy two schemas, and a model and a secret in the shared one, named after the app schema" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the app schema. The model and the secret stay in the shared schema, so only their own name changes" +trace update_file.py databricks.yml test_app_ test_app_renamed_ +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py --sort //unity-catalog + +title "The old model and secret are gone; the new ones are in the shared schema" +# Not traced: the "not found" messages differ between UC and the test server. +$CLI registered-models get "$SHARED.test_app_${UNIQUE_NAME}" &> /dev/null && echo "old model still exists" || echo "old model is gone" +$CLI secrets-uc get-secret "$SHARED.test_app_${UNIQUE_NAME}" &> /dev/null && echo "old secret still exists" || echo "old secret is gone" +trace $CLI registered-models get "$SHARED.test_app_renamed_${UNIQUE_NAME}" | jq '{full_name}' +trace $CLI secrets-uc get-secret "$SHARED.test_app_renamed_${UNIQUE_NAME}" | jq '{full_name}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/test.toml b/acceptance/bundle/resources/schemas/rename_referenced_by_name/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl index 663acec3be0..c198551c991 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl +++ b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl @@ -15,3 +15,9 @@ resources: schema_name: ${resources.schemas.schema1.name} name: myvolume comment: c1 + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt index 8218075b830..1e992b990d8 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/output.txt +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -1,15 +1,26 @@ -=== Deploy two schemas and a volume in the first one +=== Deploy two schemas, and a volume and a registered model in the first one >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 Created schemas.schema1 Created schemas.schema2 Created volumes.volume1 Files: 2 uploaded, 0 deleted -Resources: 3 created, 0 changed, 0 deleted, 0 unchanged +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --sort //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "comment": "c1", + "name": "mymodel", + "schema_name": "test-schema1-[UNIQUE_NAME]" + } +} { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", @@ -38,7 +49,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== Rename the volume's schema: the volume is re-keyed without a request +=== Rename the first schema: the volume and the model are re-keyed without a request >>> update_file.py databricks.yml test-schema1- test-schema1-renamed- @@ -46,9 +57,9 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog { "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-[UNIQUE_NAME]", @@ -59,17 +70,28 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } } -=== After the rename, update, update_id and recreate of the volume all reach it at its new path +=== After the rename, update, update_id and recreate of the volume and the model all reach them at their new paths >>> update_file.py databricks.yml comment: c1 comment: c2 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 2 changed, 0 deleted, 2 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema1-renamed-[UNIQUE_NAME].mymodel", + "body": { + "catalog_name": "main", + "comment": "c2", + "name": "mymodel", + "schema_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} { "method": "PATCH", "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", @@ -80,13 +102,26 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> update_file.py databricks.yml name: myvolume name: myvolume2 +>>> update_file.py databricks.yml name: mymodel name: mymodel2 + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 2 changed, 0 deleted, 2 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema1-renamed-[UNIQUE_NAME].mymodel", + "body": { + "catalog_name": "main", + "comment": "c2", + "new_name": "mymodel2", + "schema_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} { "method": "PATCH", "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", @@ -106,15 +141,30 @@ For managed volumes, the files stored in the volume are also deleted from your cloud tenant within 30 days. For external volumes, the metadata about the volume is removed from the catalog, but the underlying files are not deleted: recreate resources.volumes.volume1 +Recreated registered_models.model1 Recreated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 2 unchanged +Resources: 2 created, 0 changed, 2 deleted, 2 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/models/main.test-schema1-renamed-[UNIQUE_NAME].mymodel2" +} { "method": "DELETE", "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume2" } +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "comment": "c2", + "name": "mymodel2", + "schema_name": "test-schema2-[UNIQUE_NAME]" + } +} { "method": "POST", "path": "/api/2.1/unity-catalog/volumes", @@ -127,7 +177,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } } -=== Rename the volume's new schema: the volume is re-keyed again +=== Rename the second schema: the volume and the model are re-keyed again >>> update_file.py databricks.yml test-schema2- test-schema2-renamed- @@ -135,9 +185,9 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema2 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog { "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema2-[UNIQUE_NAME]", @@ -148,17 +198,28 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } } -=== Update, update_id and recreate work again, moving the volume back to the first schema +=== Update, update_id and recreate work again, moving the volume and the model back to the first schema >>> update_file.py databricks.yml comment: c2 comment: c3 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 2 changed, 0 deleted, 2 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema2-renamed-[UNIQUE_NAME].mymodel2", + "body": { + "catalog_name": "main", + "comment": "c3", + "name": "mymodel2", + "schema_name": "test-schema2-renamed-[UNIQUE_NAME]" + } +} { "method": "PATCH", "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", @@ -169,13 +230,26 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> update_file.py databricks.yml name: myvolume2 name: myvolume +>>> update_file.py databricks.yml name: mymodel2 name: mymodel + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated registered_models.model1 Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 2 changed, 0 deleted, 2 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/models/main.test-schema2-renamed-[UNIQUE_NAME].mymodel2", + "body": { + "catalog_name": "main", + "comment": "c3", + "new_name": "mymodel", + "schema_name": "test-schema2-renamed-[UNIQUE_NAME]" + } +} { "method": "PATCH", "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", @@ -195,15 +269,30 @@ For managed volumes, the files stored in the volume are also deleted from your cloud tenant within 30 days. For external volumes, the metadata about the volume is removed from the catalog, but the underlying files are not deleted: recreate resources.volumes.volume1 +Recreated registered_models.model1 Recreated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 2 unchanged +Resources: 2 created, 0 changed, 2 deleted, 2 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/models/main.test-schema2-renamed-[UNIQUE_NAME].mymodel" +} { "method": "DELETE", "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume" } +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "comment": "c3", + "name": "mymodel", + "schema_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} { "method": "POST", "path": "/api/2.1/unity-catalog/volumes", @@ -216,7 +305,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } } -=== Rename the first schema back: the volume is re-keyed to its original path +=== Rename the first schema back: the volume and the model are re-keyed to their original paths >>> update_file.py databricks.yml test-schema1-renamed- test-schema1- @@ -224,9 +313,9 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 2 unchanged +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog { "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-renamed-[UNIQUE_NAME]", @@ -241,6 +330,9 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> [CLI] bundle summary --output json { + "registered_models": { + "model1": "main.test-schema1-[UNIQUE_NAME].mymodel" + }, "schemas": { "schema1": "main.test-schema1-[UNIQUE_NAME]", "schema2": "main.test-schema2-renamed-[UNIQUE_NAME]" @@ -251,7 +343,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged >>> [CLI] volumes read main.test-schema1-[UNIQUE_NAME].myvolume { @@ -259,8 +351,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged "comment": "c3" } +>>> [CLI] registered-models get main.test-schema1-[UNIQUE_NAME].mymodel +{ + "full_name": "main.test-schema1-[UNIQUE_NAME].mymodel", + "comment": "c3" +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.registered_models.model1 delete resources.schemas.schema1 delete resources.schemas.schema2 delete resources.volumes.volume1 @@ -277,4 +376,4 @@ is removed from the catalog, but the underlying files are not deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 3 deleted +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/schemas/rename_then_change/script b/acceptance/bundle/resources/schemas/rename_then_change/script index 452af92148c..8a2579b1b5f 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/script +++ b/acceptance/bundle/resources/schemas/rename_then_change/script @@ -10,39 +10,42 @@ trap cleanup EXIT deploy() { trace $CLI bundle deploy "$@" - trace print_requests.py //unity-catalog + # The volume and the model are updated in parallel. + trace print_requests.py --sort //unity-catalog } -title "Deploy two schemas and a volume in the first one" +title "Deploy two schemas, and a volume and a registered model in the first one" trace $CLI bundle deploy -# The two schemas are created in parallel. +# The two schemas, then the volume and the model, are created in parallel. trace print_requests.py --sort //unity-catalog -title "Rename the volume's schema: the volume is re-keyed without a request" +title "Rename the first schema: the volume and the model are re-keyed without a request" trace update_file.py databricks.yml test-schema1- test-schema1-renamed- deploy -title "After the rename, update, update_id and recreate of the volume all reach it at its new path" +title "After the rename, update, update_id and recreate of the volume and the model all reach them at their new paths" trace update_file.py databricks.yml "comment: c1" "comment: c2" deploy trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +trace update_file.py databricks.yml "name: mymodel" "name: mymodel2" deploy trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name deploy --auto-approve -title "Rename the volume's new schema: the volume is re-keyed again" +title "Rename the second schema: the volume and the model are re-keyed again" trace update_file.py databricks.yml test-schema2- test-schema2-renamed- deploy -title "Update, update_id and recreate work again, moving the volume back to the first schema" +title "Update, update_id and recreate work again, moving the volume and the model back to the first schema" trace update_file.py databricks.yml "comment: c2" "comment: c3" deploy trace update_file.py databricks.yml "name: myvolume2" "name: myvolume" +trace update_file.py databricks.yml "name: mymodel2" "name: mymodel" deploy trace update_file.py databricks.yml schemas.schema2.name schemas.schema1.name deploy --auto-approve -title "Rename the first schema back: the volume is re-keyed to its original path" +title "Rename the first schema back: the volume and the model are re-keyed to their original paths" trace update_file.py databricks.yml test-schema1-renamed- test-schema1- deploy @@ -50,3 +53,4 @@ title "The state tracks the current paths and the plan is a no-op" trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' trace $CLI bundle plan trace $CLI volumes read "main.test-schema1-${UNIQUE_NAME}.myvolume" | jq '{full_name, comment}' +trace $CLI registered-models get "main.test-schema1-${UNIQUE_NAME}.mymodel" | jq '{full_name, comment}' diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt index cd987eb4ca0..18ac6b3162b 100644 --- a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt @@ -9,30 +9,24 @@ Created secrets.secret1 Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged -=== Rename the schema. UC moves the model and the secret with it; they cannot follow a rename yet, so they are recreated where they moved +=== Rename the schema. UC moves the model and the secret with it; the model follows the rename, the secret cannot yet, so it is recreated where it moved >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> [CLI] bundle plan -recreate registered_models.model1 update schemas.schema1 recreate secrets.secret1 -Plan: 2 to add, 1 to change, 2 to delete, 0 unchanged +Plan: 1 to add, 1 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Recreated registered_models.model1 Recreated secrets.secret1 Updated schemas.schema1 Files: 2 uploaded, 0 deleted -Resources: 2 created, 1 changed, 2 deleted, 0 unchanged +Resources: 1 created, 1 changed, 1 deleted, 1 unchanged >>> print_requests.py --sort //unity-catalog -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/models/main.test-schema-renamed-[UNIQUE_NAME].mymodel" -} { "method": "DELETE", "path": "/api/2.1/unity-catalog/secrets/main.test-schema-renamed-[UNIQUE_NAME].mysecret" @@ -46,15 +40,6 @@ Resources: 2 created, 1 changed, 2 deleted, 0 unchanged "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/models", - "body": { - "catalog_name": "main", - "name": "mymodel", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]" - } -} { "method": "POST", "path": "/api/2.1/unity-catalog/secrets", diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script index af95a0bb084..17140a4e99b 100644 --- a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script @@ -15,7 +15,7 @@ title "Deploy a schema with a registered model and a secret in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests -title "Rename the schema. UC moves the model and the secret with it; they cannot follow a rename yet, so they are recreated where they moved" +title "Rename the schema. UC moves the model and the secret with it; the model follows the rename, the secret cannot yet, so it is recreated where it moved" trace update_file.py databricks.yml test-schema- test-schema-renamed- trace $CLI bundle plan trace $CLI bundle deploy --auto-approve diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 5b093a61d4c..3de24687956 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -13,6 +13,8 @@ import ( "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -220,9 +222,14 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, } func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any, planEntry *deployplan.PlanEntry) error { + oldID, err := d.followParentRename(ctx, db, oldID, newState, planEntry) + if err != nil { + return err + } + var newID string var remoteState any - _, err := retry.With(ctx, retry.IsTransient, func() (struct{}, error) { + _, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) { var e error newID, remoteState, e = d.Adapter.DoUpdateWithID(ctx, oldID, newState, planEntry) return struct{}{}, e @@ -267,6 +274,57 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return nil } +// followParentRename records the resource at the ID its parent's rename moved it to, before the +// update applies the resource's own changes there. The parent is applied first, so the move has +// already happened: if the update then fails, or the deploy is interrupted, the state still points +// at the resource. Without this, the next plan would find nothing at the old ID, and a resource +// with no internal ID could not be found at its new one. It returns the ID to update at. +func (d *DeploymentUnit) followParentRename(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any, planEntry *deployplan.PlanEntry) (string, error) { + var parentPaths []*structpath.PathNode + for pathString, ch := range planEntry.Changes { + if ch.Reason != deployplan.ReasonParentRenamed { + continue + } + path, err := structpath.ParsePath(pathString) + if err != nil { + return "", err + } + parentPaths = append(parentPaths, path) + } + if len(parentPaths) == 0 { + return oldID, nil + } + + // The saved state with the parent fields of the new one: what the resource is after the move. + state, err := d.loadPersistedState(db) + if err != nil { + return "", err + } + for _, path := range parentPaths { + value, err := structaccess.Get(newState, path) + if err != nil { + return "", fmt.Errorf("reading %s: %w", path, err) + } + if err := structaccess.Set(state, path, value); err != nil { + return "", fmt.Errorf("setting %s: %w", path, err) + } + } + movedID, err := d.Adapter.IDFromState(state) + if err != nil { + return "", err + } + if movedID == oldID { + return oldID, nil + } + + log.Infof(ctx, "Re-keyed %s id=%#v (previously %#v) after its parent's rename", d.ResourceKey, movedID, oldID) + err = d.saveState(ctx, db, movedID, state, d.DependsOn, planEntry.RemoteState) + if err != nil { + return "", fmt.Errorf("saving state id=%s: %w", movedID, err) + } + return movedID, nil +} + // Rekey records the resource under the ID its new state gives it, without calling the backend: a // rename already moved it there (see PlanEntry.StateOnly). func (d *DeploymentUnit) Rekey(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any, planEntry *deployplan.PlanEntry) error { diff --git a/bundle/direct/apply_test.go b/bundle/direct/apply_test.go index 24304292ddc..d4261fc35e7 100644 --- a/bundle/direct/apply_test.go +++ b/bundle/direct/apply_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/structs/structpath" @@ -80,7 +81,7 @@ func TestUpdateWithIDFailsOnEmptyID(t *testing.T) { d := &DeploymentUnit{ResourceKey: "resources.fakes.foo", Adapter: adapter} require.NoError(t, db.SaveState(t.Context(), d.ResourceKey, "old-id", &emptyIDState{Name: "foo"}, nil)) - err = d.UpdateWithID(t.Context(), db, "old-id", &emptyIDState{Name: "bar"}, nil) + err = d.UpdateWithID(t.Context(), db, "old-id", &emptyIDState{Name: "bar"}, &deployplan.PlanEntry{}) require.EqualError(t, err, "update returned an empty resource id") assert.Equal(t, "old-id", db.GetResourceID(d.ResourceKey)) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 7e7a727ccc5..1011a0791d5 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -561,7 +561,7 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if action, reason, ok := classifyIDField(cfg, path, ch); ok { ch.Action = action ch.Reason = reason - if action == deployplan.Recreate && followsParentRename(adapter, parentRenames[pathString], ch) { + if action == deployplan.Recreate && followsParentRename(adapter, path, parentRenames[pathString], ch) { ch.Action = deployplan.UpdateWithID ch.Reason = deployplan.ReasonParentRenamed } @@ -831,8 +831,9 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { if err != nil { return err } - // The resource's own renamable name stays its own, even when it references the parent. - if isUpdatableIDField(adapter, path) { + // Only a field naming the parent moves with it: the resource's own name stays its own, + // even when it references the parent's. + if !isParentIDField(adapter, path) { continue } if err := structaccess.Set(state, path, rename.New); err != nil { @@ -862,12 +863,11 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { return nil } -// isUpdatableIDField reports whether path is one of the resource's updatable ID fields, its own -// renamable name. -func isUpdatableIDField(adapter *dresources.Adapter, path *structpath.PathNode) bool { - _, own := findMatchingRule(path, adapter.ResourceConfig().UpdatableIDFields) - _, generated := findMatchingRule(path, adapter.GeneratedResourceConfig().UpdatableIDFields) - return own || generated +// isParentIDField reports whether path is one of the resource's parent ID fields (see +// ResourceLifecycleConfig.ParentIDFields). +func isParentIDField(adapter *dresources.Adapter, path *structpath.PathNode) bool { + _, ok := findMatchingRule(path, adapter.ResourceConfig().ParentIDFields) + return ok } // recordRefTypes records, for each pure-reference field of resourceKey, the type of the resource it @@ -1077,16 +1077,17 @@ func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) return false } -// followsParentRename reports whether a change to a provided ID field only follows parentChange, -// the rename of the parent field it references (see parentRenames). UC moves the children of a -// renamed catalog or schema along with it, so recreating such a child fails with "already -// exists"; it is planned as an update_id instead, which addresses it under the new parent. That -// needs IDFromState: with no other change, the update_id only re-keys the state (StateOnly). +// followsParentRename reports whether a change to a parent ID field (see isParentIDField) only +// follows parentChange, the rename of the parent field it references (see parentRenames). UC +// moves the children of a renamed catalog or schema along with it, so recreating such a child +// fails with "already exists"; it is planned as an update_id instead, which addresses it under +// the new parent. That needs IDFromState: with no other change, the update_id only re-keys the +// state (StateOnly). // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -func followsParentRename(adapter *dresources.Adapter, parentChange, ch *deployplan.ChangeDesc) bool { - return parentChange != nil && canFollowParentRename(adapter) && structdiff.IsEqual(ch.Old, parentChange.Old) +func followsParentRename(adapter *dresources.Adapter, path *structpath.PathNode, parentChange, ch *deployplan.ChangeDesc) bool { + return parentChange != nil && isParentIDField(adapter, path) && canFollowParentRename(adapter) && structdiff.IsEqual(ch.Old, parentChange.Old) } // canFollowParentRename reports whether the resource can follow a renamed parent with an update_id: diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index a632dd11f7d..947f5bee96d 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -302,8 +302,8 @@ func TestFollowsParentRename(t *testing.T) { }, { name: "resource without DoUpdateWithID is recreated", - resource: "registered_models", - field: "schema_name", + resource: "quality_monitors", + field: "table_name", old: "s_old", expectedAction: deployplan.Recreate, }, diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 898eb90dbee..21ac7a02344 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -438,6 +438,10 @@ func (a *Adapter) validate() error { a.internalIDPath = path } + if a.resourceConfig != nil && len(a.resourceConfig.ParentIDFields) > 0 && a.idFromState == nil { + return errors.New("parent_id_fields requires IDFromState") + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f80fda21d14..745b48913c7 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1395,6 +1395,9 @@ func validateResourceConfig(t *testing.T, stateType, remoteType reflect.Type, cf for _, p := range cfg.UpdatableIDFields { assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "UpdatableIDFields: %s", p.Field) } + for _, p := range cfg.ParentIDFields { + assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "ParentIDFields: %s", p.Field) + } for _, p := range cfg.IgnoreRemoteChanges { assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "IgnoreRemoteChanges: %s", p.Field) } diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index b630b7e1176..45bb8115884 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -129,6 +129,14 @@ type ResourceLifecycleConfig struct { // the resource by its recorded ID, so it can only be backend normalization. UpdatableIDFields []FieldRule `yaml:"updatable_id_fields,omitempty"` + // ParentIDFields: the ID fields that name the resource's parent (a volume's catalog_name and + // schema_name). The backend moves the resource along with a renamed parent, so a change to one + // of these that only follows the parent's rename re-keys the resource instead of recreating it + // at its old ID (see followsParentRename and addParentMoves). Any other field, such as a name + // that references the parent's name, is the resource's own and never follows it. Requires + // IDFromState. + ParentIDFields []FieldRule `yaml:"parent_id_fields,omitempty"` + // NormalizeSlash: string field patterns the UC API strips trailing slashes from. // A change is skipped when local and remote differ only by trailing slashes. NormalizeSlash []FieldRule `yaml:"normalize_slash,omitempty"` @@ -201,6 +209,7 @@ var empty = ResourceLifecycleConfig{ RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, + ParentIDFields: nil, NormalizeSlash: nil, IgnoreRemoteAdditions: nil, BackendDefaults: nil, diff --git a/bundle/direct/dresources/configs/registered_models.yml b/bundle/direct/dresources/configs/registered_models.yml index 03bcbcc6079..598198e003e 100644 --- a/bundle/direct/dresources/configs/registered_models.yml +++ b/bundle/direct/dresources/configs/registered_models.yml @@ -20,14 +20,18 @@ ignore_remote_changes: - field: aliases[*].model_name reason: output_only provided_id_fields: - # The name can technically be updated without recreate. We recreate for now though - # to match TF implementation. - - field: name - reason: terraform_compat - field: catalog_name reason: id_field - field: schema_name reason: id_field +parent_id_fields: + - field: catalog_name + reason: parent + - field: schema_name + reason: parent +updatable_id_fields: + - field: name + reason: id_changes recreate_on_changes: - field: storage_location reason: immutable diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 5bb8f161038..51f6c69cd83 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -1,6 +1,9 @@ provided_id_fields: - field: catalog_name reason: id_field +parent_id_fields: + - field: catalog_name + reason: parent updatable_id_fields: # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - field: name diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 22ca3d7f49f..0b5918467ed 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -1,3 +1,8 @@ +parent_id_fields: + - field: catalog_name + reason: parent + - field: schema_name + reason: parent sensitive_fields: - field: value - field: effective_value diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index 9e570a1f7c2..bc001c1b773 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -3,6 +3,11 @@ provided_id_fields: reason: id_field - field: schema_name reason: id_field +parent_id_fields: + - field: catalog_name + reason: parent + - field: schema_name + reason: parent recreate_on_changes: - field: storage_location reason: immutable diff --git a/bundle/direct/dresources/registered_model.go b/bundle/direct/dresources/registered_model.go index 4352b848fe8..0186db9ae61 100644 --- a/bundle/direct/dresources/registered_model.go +++ b/bundle/direct/dresources/registered_model.go @@ -91,20 +91,27 @@ func (r *ResourceRegisteredModel) WaitAfterCreate(ctx context.Context, id string // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. var registeredModelForceSend = []string{"Comment"} -func (r *ResourceRegisteredModel) DoUpdate(ctx context.Context, id string, config *catalog.CreateRegisteredModelRequest, entry *PlanEntry) (*catalog.RegisteredModelInfo, error) { - updateRequest := catalog.UpdateRegisteredModelRequest{ - FullName: id, +func makeUpdateRegisteredModel(fullName, newName string, config *catalog.CreateRegisteredModelRequest) catalog.UpdateRegisteredModelRequest { + // UC rejects an update that sets both name and new_name, even an empty name: a config read back + // from a saved plan force-sends it. + name := config.Name + notSent := []string{"Owner", "NewName"} + if newName != "" { + name = "" + notSent = append(notSent, "Name") + } + return catalog.UpdateRegisteredModelRequest{ + FullName: fullName, Comment: config.Comment, - ForceSendFields: utils.FilterFields[catalog.UpdateRegisteredModelRequest](append(slices.Clone(registeredModelForceSend), config.ForceSendFields...), "Owner", "NewName"), + ForceSendFields: utils.FilterFields[catalog.UpdateRegisteredModelRequest](append(slices.Clone(registeredModelForceSend), config.ForceSendFields...), notSent...), // Owner is settable in the config (it comes from the embedded // CreateRegisteredModelRequest) and create sends it, but update never has: a // change to it is silently dropped rather than applied. Owner: "", - // Name updates are not supported yet without recreating. Can be added as a follow-up. - // Note: TF also does not support changing name without a recreate so the current behavior matches TF. - NewName: "", + // Set by DoUpdateWithID when the name changes. + NewName: newName, // The update API ignores aliases; syncAliases applies them through the alias API. Aliases: nil, @@ -117,27 +124,60 @@ func (r *ResourceRegisteredModel) DoUpdate(ctx context.Context, id string, confi UpdatedBy: config.UpdatedBy, SchemaName: config.SchemaName, StorageLocation: config.StorageLocation, - Name: config.Name, + Name: name, CatalogName: config.CatalogName, } +} - response, err := r.client.RegisteredModels.Update(ctx, updateRequest) +func (r *ResourceRegisteredModel) DoUpdate(ctx context.Context, id string, config *catalog.CreateRegisteredModelRequest, entry *PlanEntry) (*catalog.RegisteredModelInfo, error) { + response, err := r.client.RegisteredModels.Update(ctx, makeUpdateRegisteredModel(id, "", config)) if err != nil { return nil, err } + return response, r.syncChangedAliases(ctx, id, config, entry) +} - if entry.Changes.HasChange(registeredModelAliasesPath) { - remote, ok := entry.RemoteState.(*catalog.RegisteredModelInfo) - if !ok { - return nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) - } - err = r.syncAliases(ctx, id, config.Aliases, remote.Aliases) - if err != nil { - return nil, err - } +// DoUpdateWithID renames the model in place when the name changes, keeping its versions; the ID is +// the full name, so it changes too. +func (r *ResourceRegisteredModel) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateRegisteredModelRequest, entry *PlanEntry) (string, *catalog.RegisteredModelInfo, error) { + nameFromID, err := getNameFromID(id) + if err != nil { + return "", nil, err + } + + // The catalog and schema come from config: a renamed parent moves the model with it (see + // followsParentRename). + current := config.CatalogName + "." + config.SchemaName + "." + nameFromID + + // Aliases go first, under the current name: if they fail, the model is not renamed yet, so the + // state still points at it. + err = r.syncChangedAliases(ctx, current, config, entry) + if err != nil { + return "", nil, err } - return response, nil + newName := "" + if config.Name != nameFromID { + newName = config.Name + } + + response, err := r.client.RegisteredModels.Update(ctx, makeUpdateRegisteredModel(current, newName, config)) + if err != nil { + return "", nil, err + } + return response.FullName, response, nil +} + +// syncChangedAliases applies the configured aliases when the plan changes them. +func (r *ResourceRegisteredModel) syncChangedAliases(ctx context.Context, fullName string, config *catalog.CreateRegisteredModelRequest, entry *PlanEntry) error { + if !entry.Changes.HasChange(registeredModelAliasesPath) { + return nil + } + remote, ok := entry.RemoteState.(*catalog.RegisteredModelInfo) + if !ok { + return fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) + } + return r.syncAliases(ctx, fullName, config.Aliases, remote.Aliases) } // syncAliases makes the model's aliases match desired, given the ones it has remotely. The alias API diff --git a/libs/testserver/registered_models.go b/libs/testserver/registered_models.go index 2dea8ecc864..c41436713e5 100644 --- a/libs/testserver/registered_models.go +++ b/libs/testserver/registered_models.go @@ -28,6 +28,10 @@ func (s *FakeWorkspace) RegisteredModelsCreate(req Request) Response { // Build full name from catalog.schema.name fullName := createRequest.CatalogName + "." + createRequest.SchemaName + "." + createRequest.Name + if _, ok := s.RegisteredModels[fullName]; ok { + return registeredModelAlreadyExists(createRequest.Name) + } + registeredModel := catalog.RegisteredModelInfo{ CatalogName: createRequest.CatalogName, Comment: createRequest.Comment, @@ -58,10 +62,7 @@ func (s *FakeWorkspace) RegisteredModelsUpdate(req Request, fullName string) Res existing, ok := s.RegisteredModels[fullName] if !ok { - return Response{ - StatusCode: http.StatusNotFound, - Body: fmt.Sprintf("registered model %s not found", fullName), - } + return registeredModelNotFound(fullName) } fields, errResponse := parseUpdateFields(req.Body) @@ -87,6 +88,13 @@ func (s *FakeWorkspace) RegisteredModelsUpdate(req Request, fullName string) Res } } + if updateRequest.NewName != "" { + newFullName := existing.CatalogName + "." + existing.SchemaName + "." + updateRequest.NewName + if _, taken := s.RegisteredModels[newFullName]; taken && newFullName != fullName { + return registeredModelAlreadyExists(updateRequest.NewName) + } + } + // The update API ignores aliases; they are set through the alias endpoints. delete(fields, "aliases") applyUpdatedFields(&existing, updateRequest, fields) @@ -239,12 +247,24 @@ func (s *FakeWorkspace) RegisteredModelsDeleteAlias(fullName, aliasName string) return Response{} } +// registeredModelAlreadyExists is UC's answer to a create or rename onto a taken name; it names the +// model by its short name (verified on AWS). +func registeredModelAlreadyExists(name string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Routine or Model '%s' already exists", name), + }, + } +} + func registeredModelNotFound(fullName string) Response { return Response{ StatusCode: http.StatusNotFound, Body: map[string]string{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Registered model '%s' does not exist.", fullName), + "message": fmt.Sprintf("Routine or Model '%s' does not exist.", fullName), }, } } diff --git a/libs/testserver/registered_models_test.go b/libs/testserver/registered_models_test.go new file mode 100644 index 00000000000..835f90e0959 --- /dev/null +++ b/libs/testserver/registered_models_test.go @@ -0,0 +1,44 @@ +package testserver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRegisteredModelsCreate_RejectsExistingModel(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + request := Request{Body: []byte(`{"catalog_name": "main", "schema_name": "default", "name": "mymodel", "comment": "first"}`)} + + require.Equal(t, 0, workspace.RegisteredModelsCreate(request).StatusCode) + + response := workspace.RegisteredModelsCreate(Request{Body: []byte(`{"catalog_name": "main", "schema_name": "default", "name": "mymodel", "comment": "second"}`)}) + assert.Equal(t, 400, response.StatusCode) + assert.Equal(t, map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": "Routine or Model 'mymodel' already exists", + }, response.Body) + + // The existing model is kept as it was. + assert.Equal(t, "first", workspace.RegisteredModels["main.default.mymodel"].Comment) +} + +func TestRegisteredModelsUpdate_RejectsRenameOntoExistingModel(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + for _, name := range []string{"model_a", "model_b"} { + body := `{"catalog_name": "main", "schema_name": "default", "name": "` + name + `"}` + require.Equal(t, 0, workspace.RegisteredModelsCreate(Request{Body: []byte(body)}).StatusCode) + } + + response := workspace.RegisteredModelsUpdate(Request{Body: []byte(`{"new_name": "model_b"}`)}, "main.default.model_a") + assert.Equal(t, 400, response.StatusCode) + assert.Equal(t, map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": "Routine or Model 'model_b' already exists", + }, response.Body) + + // Both models are kept as they were. + assert.Contains(t, workspace.RegisteredModels, "main.default.model_a") + assert.Contains(t, workspace.RegisteredModels, "main.default.model_b") +}