From c715b559aa06568a14d284299d0ccceeeabd7ea3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:33:56 +0100 Subject: [PATCH 01/13] direct: find a catalog renamed outside the bundle by its id UC returns a catalog id (a UUID kept across renames, new on recreate) that the API spec and SDK do not declare. The catalog adapter now calls the catalogs API directly so it sees the id, records it in the state, and names it as the internal ID: the plan finds a catalog renamed out of band like it finds schemas and volumes. The test server returns and keeps the id too. Co-authored-by: Isaac --- .../catalogs/rename_all_out_of_band/script | 17 +-- .../catalogs/rename_out_of_band/script | 17 +-- .../resources/grants/catalogs/out.plan2.json | 9 ++ bundle/direct/dresources/catalog.go | 103 ++++++++++++++---- bundle/direct/dresources/configs/catalogs.yml | 3 + libs/testserver/catalogs.go | 24 +++- libs/testserver/catalogs_test.go | 21 +++- libs/testserver/fake_workspace.go | 4 +- 8 files changed, 153 insertions(+), 45 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index aa51aadefb6..26fed081e3c 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -5,8 +5,6 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace $CLI catalogs delete "$NEW_CATALOG" --force # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -21,6 +19,8 @@ plan() { title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests +# The SDK drops the catalog's id, so read it with a raw GET. +add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID @@ -34,11 +34,14 @@ title "Rename all three in the config to match" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ trace update_file.py databricks.yml "name: myschema" "name: myschema2" trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" -NEW_SCHEMA="$NEW_CATALOG.myschema2" -NEW_VOLUME="$NEW_SCHEMA.myvolume2" -title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +title "The plan finds the catalog, the schema and the volume at their new names by their ids" plan -title "The catalog's create fails because the renamed catalog has its name" -trace musterr $CLI bundle deploy +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index 1c39637d66b..d94cb0278ff 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -5,8 +5,6 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace $CLI catalogs delete "$NEW_CATALOG" --force # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -21,6 +19,8 @@ plan() { title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests +# The SDK drops the catalog's id, so read it with a raw GET. +add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID @@ -30,11 +30,14 @@ print_requests.py //unity-catalog &> LOG.rename_requests title "Rename the catalog in the config to match" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -NEW_SCHEMA="$NEW_CATALOG.myschema" -NEW_VOLUME="$NEW_SCHEMA.myvolume" -title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +title "The plan finds the catalog, the schema and the volume at their new names by their ids" plan -title "The catalog's create fails because the renamed catalog has its name" -trace musterr $CLI bundle deploy +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.json index 92d227bc232..9850835017c 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.json @@ -10,8 +10,17 @@ "catalog_type": "MANAGED_CATALOG", "comment": "Test catalog for grants", "full_name": "catalog_grants_[UNIQUE_NAME]", + "id": "[UUID]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.catalogs.grants_catalog.grants": { diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 53d9b3b23f1..02b808c3386 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -3,23 +3,52 @@ package dresources import ( "context" "fmt" + "maps" + "net/http" "slices" + "sync" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +const catalogsPath = "/api/2.1/unity-catalog/catalogs" + +// CatalogRemote is CatalogInfo plus the catalog's id, a UUID that UC returns but the API spec does +// not declare, so the SDK drops it. A rename keeps the id and a recreate changes it (verified on +// AWS, Azure and GCP). +type CatalogRemote struct { + catalog.CatalogInfo + + Id string `json:"id,omitempty"` +} + +// Custom marshaler needed because embedded CatalogInfo has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *CatalogRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s CatalogRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // CatalogState is CreateCatalog plus isolation_mode, which the Create API does not -// accept; DoCreate applies it with a follow-up update. +// accept (DoCreate applies it with a follow-up update), and the catalog's id. type CatalogState struct { catalog.CreateCatalog IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + + // Recorded by the framework from the remote state (internal_id_field). + Id string `json:"id,omitempty"` } -// Custom marshalers needed because embedded catalog.CreateCatalog has its own MarshalJSON +// Custom marshaler needed because embedded CreateCatalog has its own MarshalJSON // which would otherwise take over and ignore the additional fields. func (s *CatalogState) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) @@ -30,22 +59,50 @@ func (s CatalogState) MarshalJSON() ([]byte, error) { } type ResourceCatalog struct { - client *databricks.WorkspaceClient + client *databricks.WorkspaceClient + apiClient func() (*client.DatabricksClient, error) +} + +func (*ResourceCatalog) New(w *databricks.WorkspaceClient) *ResourceCatalog { + return &ResourceCatalog{ + client: w, + apiClient: sync.OnceValues(func() (*client.DatabricksClient, error) { + return client.New(w.Config) + }), + } } -func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog { - return &ResourceCatalog{client: client} +// do calls the catalogs API like the SDK does, but decodes the response into CatalogRemote: +// the SDK's CatalogInfo has no id. +func (r *ResourceCatalog) do(ctx context.Context, method, path string, request any) (*CatalogRemote, error) { + apiClient, err := r.apiClient() + if err != nil { + return nil, err + } + headers := map[string]string{"Accept": "application/json"} + if request != nil { + headers["Content-Type"] = "application/json" + } + maps.Copy(headers, auth.WorkspaceIDHeaders(r.client.Config)) + var response CatalogRemote + err = apiClient.Do(ctx, method, path, headers, nil, request, &response) + if err != nil { + return nil, err + } + return &response, nil } func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { return &CatalogState{ CreateCatalog: input.CreateCatalog, IsolationMode: input.IsolationMode, + Id: "", // Recorded by the framework from the remote state (internal_id_field). } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { - return &CatalogState{ +func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { + state := &CatalogState{IsolationMode: info.IsolationMode, Id: info.Id} + state.CreateCatalog = catalog.CreateCatalog{ Comment: info.Comment, ConnectionName: info.ConnectionName, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -57,17 +114,17 @@ func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { ShareName: info.ShareName, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), - IsolationMode: info.IsolationMode, } + return state } -func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.CatalogInfo, error) { - return r.client.Catalogs.GetByName(ctx, id) +func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*CatalogRemote, error) { + return r.do(ctx, http.MethodGet, catalogsPath+"/"+id, nil) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) - if err != nil || response == nil { +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *CatalogRemote, error) { + response, err := r.do(ctx, http.MethodPost, catalogsPath, config.CreateCatalog) + if err != nil { return "", nil, err } if config.IsolationMode == "" { @@ -75,7 +132,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s } // UC only accepts isolation_mode on update, so a configured value takes a second call. - response, err = r.client.Catalogs.Update(ctx, catalog.UpdateCatalog{ + response, err = r.do(ctx, http.MethodPatch, catalogsPath+"/"+response.Name, catalog.UpdateCatalog{ Comment: "", CustomMaxRetentionHours: 0, EnablePredictiveOptimization: "", @@ -96,6 +153,11 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s return response.Name, response, nil } +// IDFromState returns the name, which is the ID. +func (*ResourceCatalog) IDFromState(state *CatalogState) string { + return state.Name +} + // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. // // CustomMaxRetentionHours is force-sent here even though schemaForceSend deliberately omits it: @@ -105,7 +167,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} // DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*CatalogRemote, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, @@ -120,16 +182,11 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *Catal ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } - response, err := r.client.Catalogs.Update(ctx, updateRequest) - if err != nil { - return nil, err - } - - return response, nil + return r.do(ctx, http.MethodPatch, catalogsPath+"/"+id, updateRequest) } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *CatalogRemote, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, @@ -148,7 +205,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config updateRequest.NewName = config.Name } - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.do(ctx, http.MethodPatch, catalogsPath+"/"+id, updateRequest) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/configs/catalogs.yml b/bundle/direct/dresources/configs/catalogs.yml index c1e1893af42..346d529795e 100644 --- a/bundle/direct/dresources/configs/catalogs.yml +++ b/bundle/direct/dresources/configs/catalogs.yml @@ -24,3 +24,6 @@ backend_defaults: # value alone (OPEN by default, or ISOLATED when set outside the bundle) rather than # forcing it back to OPEN. - field: isolation_mode +# Assigned by the backend and kept across renames: lets the plan find a catalog renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: id diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 53740b3bd89..7479dd0f8d7 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -6,9 +6,26 @@ import ( "net/http" "strings" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// fakeCatalog is CatalogInfo plus the id UC returns, which its API spec does not declare. The id +// is kept across renames. +type fakeCatalog struct { + catalog.CatalogInfo + + Id string `json:"id,omitempty"` +} + +func (c *fakeCatalog) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c fakeCatalog) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + // catalogNameManagedDefaults scopes the managed-property injection to the // backend-default drift test, keeping unrelated tests free of it. Mirrors // schemaNameManagedDefaults. @@ -85,9 +102,10 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } } - s.Catalogs[createRequest.Name] = catalogInfo + created := fakeCatalog{CatalogInfo: catalogInfo, Id: nextUUID()} + s.Catalogs[createRequest.Name] = created return Response{ - Body: catalogInfo, + Body: created, } } @@ -115,7 +133,7 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { } } - applyUpdatedFields(&existing, updateRequest, fields) + applyUpdatedFields(&existing.CatalogInfo, updateRequest, fields) if updateRequest.NewName != "" { existing.Name = updateRequest.NewName existing.FullName = updateRequest.NewName diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index de924c08060..7ea99ee71a2 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -38,7 +38,7 @@ func TestCatalogsCreate_AllowsNonEmptyName(t *testing.T) { getResponse := MapGet(workspace, workspace.Catalogs, "my_catalog") require.Equal(t, 0, getResponse.StatusCode) - body, ok := getResponse.Body.(catalog.CatalogInfo) + body, ok := getResponse.Body.(fakeCatalog) require.True(t, ok) assert.Equal(t, "my_catalog", body.Name) assert.Equal(t, "my_catalog", body.FullName) @@ -78,8 +78,9 @@ func TestCatalogsCreate_EchoesEveryAcceptedField(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) assert.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + body, ok := response.Body.(fakeCatalog) require.True(t, ok) + info := body.CatalogInfo assert.Equal(t, "my_catalog", info.Name) assert.Equal(t, "c", info.Comment) @@ -136,8 +137,9 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { }`)}, "my_catalog") assert.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + body, ok := response.Body.(fakeCatalog) require.True(t, ok) + info := body.CatalogInfo assert.Equal(t, "updated", info.Comment) assert.Equal(t, int64(72), info.CustomMaxRetentionHours) @@ -147,3 +149,16 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { assert.Equal(t, map[string]string{"opt": "3"}, info.Options) assert.Equal(t, map[string]string{"prop": "4"}, info.Properties) } + +func TestCatalogsUpdate_RenameKeepsID(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + + created, ok := workspace.CatalogsCreate(Request{Body: []byte(`{"name": "my_catalog"}`)}).Body.(fakeCatalog) + require.True(t, ok) + require.NotEmpty(t, created.Id) + + renamed, ok := workspace.CatalogsUpdate(Request{Body: []byte(`{"new_name": "my_catalog2"}`)}, "my_catalog").Body.(fakeCatalog) + require.True(t, ok) + assert.Equal(t, "my_catalog2", renamed.Name) + assert.Equal(t, created.Id, renamed.Id) +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5e8068b6d02..338e4870a9c 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -242,7 +242,7 @@ type FakeWorkspace struct { ClusterLibraries map[string][]compute.Library // cluster id -> installed libraries InstancePools map[string]compute.GetInstancePool ClusterPolicies map[string]compute.Policy - Catalogs map[string]catalog.CatalogInfo + Catalogs map[string]fakeCatalog ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo ModelServices map[string]catalog.ModelService @@ -538,7 +538,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, - Catalogs: map[string]catalog.CatalogInfo{}, + Catalogs: map[string]fakeCatalog{}, ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, From 2ac2ea3f662b8b0914dcb5b605de64302b596bfc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:34:29 +0100 Subject: [PATCH 02/13] acc: drop the catalog out-of-band Badness this fixes Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename_all_out_of_band/test.toml | 1 - .../bundle/resources/catalogs/rename_out_of_band/test.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml index 698c1c6ca49..4272a80cf61 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -1,4 +1,3 @@ -Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true Ignore = [ diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml index 698c1c6ca49..4272a80cf61 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -1,4 +1,3 @@ -Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true Ignore = [ From 9d1dd726f78fbeb0b32dca99b97e1aff0e41d5f1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:36:38 +0100 Subject: [PATCH 03/13] direct: build the catalog state in one literal Co-authored-by: Isaac --- bundle/direct/dresources/catalog.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 02b808c3386..933c580c2ba 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -101,8 +101,7 @@ func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { } func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { - state := &CatalogState{IsolationMode: info.IsolationMode, Id: info.Id} - state.CreateCatalog = catalog.CreateCatalog{ + return &CatalogState{ Comment: info.Comment, ConnectionName: info.ConnectionName, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -114,8 +113,9 @@ func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { ShareName: info.ShareName, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), + IsolationMode: info.IsolationMode, + Id: info.Id, } - return state } func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*CatalogRemote, error) { From cc2364cad2fb627b4223f8457496b7cb90217d5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:39:20 +0100 Subject: [PATCH 04/13] Add changelog fragment for the catalog id lookup Co-authored-by: Isaac --- .nextchanges/bundles/catalog-renamed-out-of-band.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/catalog-renamed-out-of-band.md diff --git a/.nextchanges/bundles/catalog-renamed-out-of-band.md b/.nextchanges/bundles/catalog-renamed-out-of-band.md new file mode 100644 index 00000000000..9ca1bd630c8 --- /dev/null +++ b/.nextchanges/bundles/catalog-renamed-out-of-band.md @@ -0,0 +1 @@ +* A catalog renamed outside the bundle is now found by its id, instead of being created again (which failed with `CATALOG_ALREADY_EXISTS`). ([#7418](https://github.com/databricks/cli/pull/7418)) From 2e6b847ede03c5933936c9a83097148a6e25d4e3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 22:05:19 +0100 Subject: [PATCH 05/13] acc: regenerate refschema for the catalog id Co-authored-by: Isaac --- acceptance/bundle/refschema/output/catalogs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/refschema/output/catalogs.txt b/acceptance/bundle/refschema/output/catalogs.txt index 42fb0f78ead..df74c0d7910 100644 --- a/acceptance/bundle/refschema/output/catalogs.txt +++ b/acceptance/bundle/refschema/output/catalogs.txt @@ -11,7 +11,7 @@ resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type resources.catalogs.*.effective_predictive_optimization_flag.value catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.full_name string REMOTE -resources.catalogs.*.id string INPUT +resources.catalogs.*.id string ALL resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.depends_on []resources.DependsOn INPUT From c5c8c7a025c77d6bf02bcfffd20b7761d5d29c0e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 17:05:18 +0100 Subject: [PATCH 06/13] acc: also deploy the catalog out-of-band tests from a saved plan (READPLAN) Co-authored-by: Isaac --- .../catalogs/rename_all_out_of_band/out.test.toml | 1 + .../resources/catalogs/rename_all_out_of_band/script | 4 ++-- .../resources/catalogs/rename_all_out_of_band/test.toml | 7 +++++++ .../resources/catalogs/rename_out_of_band/out.test.toml | 1 + .../bundle/resources/catalogs/rename_out_of_band/script | 4 ++-- .../bundle/resources/catalogs/rename_out_of_band/test.toml | 7 +++++++ 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index 26fed081e3c..369c17d4a80 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -13,7 +13,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location } title "Deploy a catalog with a schema and a volume in it" @@ -39,7 +39,7 @@ title "The plan finds the catalog, the schema and the volume at their new names plan title "Deploy only records the new names in the state" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index d94cb0278ff..4032ce015fc 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -13,7 +13,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location } title "Deploy a catalog with a schema and a volume in it" @@ -35,7 +35,7 @@ title "The plan finds the catalog, the schema and the volume at their new names plan title "Deploy only records the new names in the state" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' From 5c73f4b36a9e070c0dac4f867aafce48fad9df8b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:36:28 +0100 Subject: [PATCH 07/13] Make the changelog entry mirror the PR title Co-authored-by: Isaac --- .nextchanges/bundles/catalog-renamed-out-of-band.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/catalog-renamed-out-of-band.md b/.nextchanges/bundles/catalog-renamed-out-of-band.md index 9ca1bd630c8..1f582e5c5e5 100644 --- a/.nextchanges/bundles/catalog-renamed-out-of-band.md +++ b/.nextchanges/bundles/catalog-renamed-out-of-band.md @@ -1 +1 @@ -* A catalog renamed outside the bundle is now found by its id, instead of being created again (which failed with `CATALOG_ALREADY_EXISTS`). ([#7418](https://github.com/databricks/cli/pull/7418)) +* Find a catalog renamed outside the bundle by its id. ([#7418](https://github.com/databricks/cli/pull/7418)) From 333b0f559c684f46c5df83bb7d958b191ea82454 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:54:33 +0100 Subject: [PATCH 08/13] acc: regenerate the catalog goldens after rebase Co-authored-by: Isaac --- .../catalogs/comment_out_of_band/output.txt | 6 ++ .../drift/managed_properties/output.txt | 6 ++ .../catalogs/isolation_mode/output.txt | 6 ++ .../rename_all_out_of_band/output.txt | 80 ++++++++++++++----- .../catalogs/rename_out_of_band/output.txt | 80 ++++++++++++++----- .../resources/grants/catalogs/out.plan2.json | 8 +- 6 files changed, 142 insertions(+), 44 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt index 5da6c37fe2a..64b652fe1d7 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt @@ -17,6 +17,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "action": "update", "remote": "set outside the bundle" }, + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt index 886246cec10..cfbb45cb47d 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt @@ -16,6 +16,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan --output json { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt index c37befe061b..4f03b1b9c15 100644 --- a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt +++ b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt @@ -80,6 +80,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "reason": "empty", "remote": 0 }, + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index af5dcff5840..9e9f3e62aa0 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -34,23 +34,41 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml name: myvolume name: myvolume2 -=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +=== The plan finds the catalog, the schema and the volume at their new names by their ids >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "action": "create", + "id": "test_catalog_renamed_[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "name": "test_catalog_renamed_[UNIQUE_NAME]" } }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME]", + "id": "[CATALOG_ID]", + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[CATALOG_ID]", + "remote": "[CATALOG_ID]" + }, "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]" + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" } } }, @@ -73,7 +91,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", "name": "myschema2", @@ -127,7 +145,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][1], + "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", "name": "myvolume2", @@ -168,26 +186,48 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== The catalog's create fails because the renamed catalog has its name +=== Deploy only records the new names in the state ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs -HTTP Status: 400 Bad Request -API error_code: CATALOG_ALREADY_EXISTS -API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists +>>> print_requests.py //unity-catalog -Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2" + } +} -Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 +=== Redeploy is a no-op -Files: 3 uploaded, 0 deleted +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 -Destroy: 0 deleted +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default ->>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index cf94276e0cd..75997e56b47 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -20,23 +20,41 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +=== The plan finds the catalog, the schema and the volume at their new names by their ids >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "action": "create", + "id": "test_catalog_renamed_[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "name": "test_catalog_renamed_[UNIQUE_NAME]" } }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME]", + "id": "[CATALOG_ID]", + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[CATALOG_ID]", + "remote": "[CATALOG_ID]" + }, "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]" + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" } } }, @@ -59,7 +77,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", "name": "myschema", @@ -106,7 +124,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][1], + "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", "name": "myvolume", @@ -133,26 +151,48 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== The catalog's create fails because the renamed catalog has its name +=== Deploy only records the new names in the state ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs -HTTP Status: 400 Bad Request -API error_code: CATALOG_ALREADY_EXISTS -API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists +>>> print_requests.py //unity-catalog -Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume" + } +} -Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 +=== Redeploy is a no-op -Files: 3 uploaded, 0 deleted +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 -Destroy: 0 deleted +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default ->>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.json index 9850835017c..49cd77a9705 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.json @@ -1,7 +1,7 @@ { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 1, "plan": { "resources.catalogs.grants_catalog": { @@ -10,7 +10,7 @@ "catalog_type": "MANAGED_CATALOG", "comment": "Test catalog for grants", "full_name": "catalog_grants_[UNIQUE_NAME]", - "id": "[UUID]", + "id": "[UUID][1]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" }, @@ -18,8 +18,8 @@ "id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, From 193b9d4024252e6e83b5921caa3d04469cf98c2e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:59:10 +0100 Subject: [PATCH 09/13] testserver: the isolation_mode test reads the created catalog as fakeCatalog Co-authored-by: Isaac --- libs/testserver/catalogs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index 7ea99ee71a2..ebd071dc9f0 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -115,7 +115,7 @@ func TestCatalogsCreate_DefaultsIsolationModeToOpen(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) require.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + info, ok := response.Body.(fakeCatalog) require.True(t, ok) assert.Equal(t, catalog.CatalogIsolationModeOpen, info.IsolationMode) } From 768805bfd7d20d98e05f56025457a53f20b056b0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 11:11:41 +0100 Subject: [PATCH 10/13] acc: show the catalog id in the model, secret and volume rename tests Co-authored-by: Isaac --- .../secrets/parent_rename/output.txt | 28 +++++++++++++++---- .../recreate_in_renamed_schema/output.txt | 15 ++++++++-- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 6cac5f13385..3d886f935f3 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -49,8 +49,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } } }, "resources.schemas.schema1": { @@ -74,7 +83,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "name": { @@ -87,8 +96,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, @@ -225,10 +234,17 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + }, "name": { "action": "update_id", "reason": "id_changes", @@ -262,7 +278,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema2", "name": "myschema2", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "catalog_name": { @@ -285,8 +301,8 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index c292b659090..1968278597b 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -23,8 +23,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } } }, "resources.schemas.schema1": { @@ -48,7 +57,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "name": { @@ -61,8 +70,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, From 6c817da315ef3b55993ccf4b4605e5f8edb9c179 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 14:27:54 +0100 Subject: [PATCH 11/13] acc: show the catalog id in the catalog rename tests that now have a model Co-authored-by: Isaac --- acceptance/bundle/resources/catalogs/rename/output.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 5bb6c4b2655..12894db3bbb 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -65,10 +65,17 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "name": { "action": "update_id", "reason": "id_changes", From 26a83220d9b24aac621a920b2433b725181a2e42 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 16:43:16 +0100 Subject: [PATCH 12/13] acc: show the catalog id in the catalog and schema rename tests after rebase Co-authored-by: Isaac --- .../rename_catalog_and_schema/output.txt | 13 ++++++++++--- .../volumes/rename_catalog_and_schema/output.txt | 7 +++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt index 895e3dce8a8..5e38a51622e 100644 --- a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt @@ -30,10 +30,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + }, "name": { "action": "update_id", "reason": "id_changes", @@ -124,7 +131,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "catalog_name": { @@ -144,8 +151,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } } diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt index f5d7df1a699..621819f3180 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -31,10 +31,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "name": { "action": "update_id", "reason": "id_changes", From 68d90206a213cf7d0472cf6e1f99f333949fdb46 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Nov 2026 14:15:10 +0100 Subject: [PATCH 13/13] acc: restore the cleanup that deletes a leaked renamed catalog (Fable review) If a cloud run fails before the plan finds the renamed catalog, destroy looks at the old name and the catalog, with its schema and volume, would leak into the shared workspace. Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename_all_out_of_band/script | 3 +++ acceptance/bundle/resources/catalogs/rename_out_of_band/script | 3 +++ 2 files changed, 6 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index 369c17d4a80..a0ce8b2754c 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -5,6 +5,9 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve + # A run that fails before the plan finds the renamed catalog leaves it where destroy does not + # look. Normally destroy has deleted it already, so this fails quietly. + errcode $CLI catalogs delete "$NEW_CATALOG" --force &> LOG.cleanup_catalog # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index 4032ce015fc..ad34b43a367 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -5,6 +5,9 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve + # A run that fails before the plan finds the renamed catalog leaves it where destroy does not + # look. Normally destroy has deleted it already, so this fails quietly. + errcode $CLI catalogs delete "$NEW_CATALOG" --force &> LOG.cleanup_catalog # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests