Skip to content

Commit 19dde68

Browse files
denikIsaac
andcommitted
Build on the lower stack layers: typed mutator versions, tests and generated code for the typed configuration
Co-authored-by: Isaac <no-reply@databricks.com>
1 parent ee33901 commit 19dde68

14 files changed

Lines changed: 2438 additions & 360 deletions

‎acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11

22
>>> [CLI] bundle deploy
3+
Warning: required field "quartz_cron_expression" is not set
4+
at resources.quality_monitors.dst.schedule
5+
in databricks.yml:39:21
6+
databricks.yml:30:17
7+
8+
Warning: required field "timezone_id" is not set
9+
at resources.quality_monitors.dst.schedule
10+
in databricks.yml:39:21
11+
databricks.yml:30:17
12+
313
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files...
414
Created quality_monitors.dst
515
Created quality_monitors.src

‎bundle/config/loader/process_root_includes_test.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,6 @@ func TestProcessRootIncludesKeepsIncludesOfIncludedFiles(t *testing.T) {
162162
diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes())
163163
require.NoError(t, diags.Error())
164164
assert.Equal(t, []string{"a.yml", "b.yml", "c.yml"}, b.Config.Include)
165-
assert.Equal(t, []any{"a.yml", "b.yml", "c.yml"}, b.Config.Value().Get("include").AsAny())
166165
}
167166

168167
func TestProcessRootIncludesNotExists(t *testing.T) {

‎bundle/config/mutator/prepend_workspace_prefix.go‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package mutator
22

33
import (
44
"context"
5-
"slices"
65
"strings"
76

87
"github.com/databricks/cli/bundle"
@@ -27,16 +26,30 @@ var skipPrefixes = []string{
2726

2827
func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
2928
w := &b.Config.Workspace
30-
for _, path := range []*string{&w.RootPath, &w.FilePath, &w.ArtifactPath, &w.StatePath, &w.ResourcePath} {
31-
// Skip prefixing if the path does not start with /, it might be variable reference or smth else.
32-
if !strings.HasPrefix(*path, "/") {
33-
continue
34-
}
35-
if slices.ContainsFunc(skipPrefixes, func(prefix string) bool { return strings.HasPrefix(*path, prefix) }) {
36-
continue
37-
}
38-
// The value keeps its locations, so diagnostics point at the original config line.
39-
*path = "/Workspace" + *path
29+
for _, path := range []*string{
30+
&w.RootPath,
31+
&w.FilePath,
32+
&w.ArtifactPath,
33+
&w.StatePath,
34+
&w.ResourcePath,
35+
} {
36+
*path = prependWorkspacePrefixTo(*path)
4037
}
38+
4139
return nil
4240
}
41+
42+
func prependWorkspacePrefixTo(path string) string {
43+
// Skip prefixing if the path does not start with /, it might be variable reference or smth else.
44+
if !strings.HasPrefix(path, "/") {
45+
return path
46+
}
47+
48+
for _, prefix := range skipPrefixes {
49+
if strings.HasPrefix(path, prefix) {
50+
return path
51+
}
52+
}
53+
54+
return "/Workspace" + path
55+
}

‎bundle/config/mutator/resourcemutator/apply_bundle_permissions.go‎

Lines changed: 49 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@ package resourcemutator
33
import (
44
"context"
55
"fmt"
6+
"maps"
7+
"reflect"
68
"slices"
79
"strings"
810

911
"github.com/databricks/cli/bundle/permissions"
1012

1113
"github.com/databricks/cli/bundle"
12-
"github.com/databricks/cli/bundle/config"
1314
"github.com/databricks/cli/bundle/config/resources"
1415
"github.com/databricks/cli/libs/diag"
1516
"github.com/databricks/cli/libs/logdiag"
16-
"github.com/databricks/cli/libs/structs/structpath"
1717
"github.com/databricks/databricks-sdk-go/service/iam"
1818
)
1919

@@ -108,44 +108,63 @@ func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Di
108108
return diag.FromErr(err)
109109
}
110110

111-
b.Config.Resources.IterateAllPermissions(func(typ, name string, perms config.PermissionList) bool {
112-
lm, ok := levelsMap[typ]
111+
for _, group := range b.Config.Resources.AllResources() {
112+
key := group.Description.PluralName
113+
lm, ok := levelsMap[key]
113114
if !ok {
114-
return true
115+
continue
115116
}
116-
err = applyToResource(ctx, b, typ, name, lm, perms)
117-
return err == nil
118-
})
119-
return diag.FromErr(err)
120-
}
121117

122-
// applyToResource applies the bundle permissions to the permissions perms of the
123-
// resource name of type typ, using the permission levels lm of that type.
124-
func applyToResource(ctx context.Context, b *bundle.Bundle, typ, name string, lm map[string]string, perms config.PermissionList) error {
125-
path := structpath.NewPath(nil, "resources", typ, name, "permissions")
126-
// Present as [structvar.View] decides: a non-nil list or a pure reference.
127-
had := !perms.IsNil() || b.Config.IsReferenceAt(path)
128-
129-
existing := make([]resources.Permission, 0, perms.Len())
130-
for i := range perms.Len() {
131-
existing = append(existing, perms.At(i))
118+
for _, name := range slices.Sorted(maps.Keys(group.Resources)) {
119+
field := reflect.ValueOf(group.Resources[name]).Elem().FieldByName("Permissions")
120+
permissions := readPermissions(field)
121+
permissions = append(permissions, convertPermissions(
122+
ctx,
123+
b.Config.Permissions,
124+
permissions,
125+
key,
126+
lm,
127+
)...)
128+
writePermissions(field, permissions)
129+
}
132130
}
133131

134-
added := convertPermissions(ctx, b.Config.Permissions, existing, typ, lm)
135-
for _, p := range added {
136-
perms.Append(p)
132+
return nil
133+
}
134+
135+
// readPermissions converts the permissions field of a resource to []resources.Permission.
136+
// The element type of the field differs between resources, but all have the same fields.
137+
func readPermissions(field reflect.Value) []resources.Permission {
138+
var permissions []resources.Permission
139+
for i := range field.Len() {
140+
p := field.Index(i)
141+
permissions = append(permissions, resources.Permission{
142+
Level: iam.PermissionLevel(p.FieldByName("Level").String()),
143+
UserName: p.FieldByName("UserName").String(),
144+
ServicePrincipalName: p.FieldByName("ServicePrincipalName").String(),
145+
GroupName: p.FieldByName("GroupName").String(),
146+
})
137147
}
148+
return permissions
149+
}
138150

139-
if !had {
140-
return nil
151+
// writePermissions sets the permissions field of a resource to permissions.
152+
// An empty list is stored as nil, so that the field is absent from the configuration.
153+
func writePermissions(field reflect.Value, permissions []resources.Permission) {
154+
if len(permissions) == 0 {
155+
field.SetZero()
156+
return
141157
}
142158

143-
// Empty permissions are dropped. Otherwise they are rebuilt without locations.
144-
if len(existing) == 0 && len(added) == 0 {
145-
return b.Config.Delete(path)
159+
out := reflect.MakeSlice(field.Type(), len(permissions), len(permissions))
160+
for i, p := range permissions {
161+
e := out.Index(i)
162+
e.FieldByName("Level").SetString(string(p.Level))
163+
e.FieldByName("UserName").SetString(p.UserName)
164+
e.FieldByName("ServicePrincipalName").SetString(p.ServicePrincipalName)
165+
e.FieldByName("GroupName").SetString(p.GroupName)
146166
}
147-
b.Config.SetLocations(path, nil)
148-
return nil
167+
field.Set(out)
149168
}
150169

151170
func validatePermissions(b *bundle.Bundle) error {

‎bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"github.com/databricks/cli/bundle/config"
1212
"github.com/databricks/cli/bundle/config/resources"
1313
"github.com/databricks/cli/libs/diag"
14-
"github.com/databricks/cli/libs/dyn"
1514
"github.com/databricks/databricks-sdk-go/service/jobs"
1615
"github.com/stretchr/testify/assert"
1716
"github.com/stretchr/testify/require"
@@ -262,8 +261,5 @@ func TestApplyBundlePermissionsEmpty(t *testing.T) {
262261
// An empty list of permissions is dropped, an unset one stays unset.
263262
for _, name := range []string{"empty", "unset"} {
264263
require.Nil(t, b.Config.Resources.Jobs[name].Permissions)
265-
v, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.jobs."+name))
266-
require.NoError(t, err)
267-
assert.False(t, v.Get("permissions").IsValid(), name)
268264
}
269265
}

‎bundle/config/mutator/resourcemutator/capture_uc_dependencies.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia
194194
if f == nil {
195195
continue
196196
}
197-
if dynvar.ContainsVariableReference(f.FullName) {
197+
if structvar.ContainsVariableReference(f.FullName) {
198198
continue
199199
}
200200
parts, ok := splitUCName(f.FullName, 3)

‎bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go‎

Lines changed: 20 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@ package resourcemutator
22

33
import (
44
"context"
5-
"errors"
6-
"maps"
75
"path/filepath"
8-
"slices"
96

107
"github.com/databricks/cli/bundle"
118
"github.com/databricks/cli/bundle/libraries"
129
"github.com/databricks/cli/libs/diag"
1310
"github.com/databricks/cli/libs/patchwheel"
14-
"github.com/databricks/cli/libs/structs/structpath"
1511
"github.com/databricks/databricks-sdk-go/service/pipelines"
1612
)
1713

@@ -21,18 +17,31 @@ func ExpandPipelineGlobPaths() bundle.Mutator {
2117
return &expandPipelineGlobPaths{}
2218
}
2319

24-
// expandLibrary returns the libraries lib is replaced with.
20+
// expandLibrary returns the libraries that replace lib: one per match of its glob path.
2521
func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, lib pipelines.PipelineLibrary) ([]pipelines.PipelineLibrary, error) {
2622
// Probe for the path field in the library.
2723
var path string
24+
var withPath func(path string) pipelines.PipelineLibrary
2825
switch {
2926
case lib.Notebook != nil && lib.Notebook.Path != "":
3027
path = lib.Notebook.Path
28+
withPath = func(path string) pipelines.PipelineLibrary {
29+
nl, notebook := lib, *lib.Notebook
30+
notebook.Path = path
31+
nl.Notebook = &notebook
32+
return nl
33+
}
3134
case lib.File != nil && lib.File.Path != "":
3235
path = lib.File.Path
36+
withPath = func(path string) pipelines.PipelineLibrary {
37+
nl, file := lib, *lib.File
38+
file.Path = path
39+
nl.File = &file
40+
return nl
41+
}
3342
default:
34-
// Neither of the library paths were found (or the path is empty). This is likely an invalid node,
35-
// but it isn't this mutator's job to enforce that. Return the original value.
43+
// Neither of the library paths were found (or the path is empty). This is likely
44+
// an invalid node, but it isn't this mutator's job to enforce that. Return the original value.
3645
return []pipelines.PipelineLibrary{lib}, nil
3746
}
3847

@@ -60,58 +69,28 @@ func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string,
6069
if err != nil {
6170
return nil, err
6271
}
63-
nl := lib
64-
if lib.Notebook != nil {
65-
notebook := *lib.Notebook
66-
notebook.Path = filepath.ToSlash(m)
67-
nl.Notebook = &notebook
68-
} else {
69-
file := *lib.File
70-
file.Path = filepath.ToSlash(m)
71-
nl.File = &file
72-
}
73-
expanded = append(expanded, nl)
72+
expanded = append(expanded, withPath(filepath.ToSlash(m)))
7473
}
7574

7675
return expanded, nil
7776
}
7877

7978
func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
80-
pls := b.Config.Resources.Pipelines
81-
for _, name := range slices.Sorted(maps.Keys(pls)) {
82-
pipeline := pls[name]
83-
if pipeline == nil {
84-
continue
85-
}
86-
path := structpath.NewPath(nil, "resources", "pipelines", name, "libraries")
87-
if b.Config.IsReferenceAt(path) {
88-
return diag.FromErr(errors.New("expected sequence, got string"))
89-
}
90-
// An explicit `libraries: null` is forced; an omitted one is not.
91-
if pipeline.Libraries == nil && slices.Contains(pipeline.ForceSendFields, "Libraries") {
92-
return diag.FromErr(errors.New("expected sequence, got nil"))
93-
}
79+
// Visit each pipeline's "libraries" field and expand any glob patterns.
80+
for _, pipeline := range b.Config.Resources.Pipelines {
9481
if len(pipeline.Libraries) == 0 {
9582
continue
9683
}
9784

98-
// Expand any glob patterns in the pipeline's libraries.
9985
var expanded []pipelines.PipelineLibrary
100-
var sources [][]int
101-
for i, lib := range pipeline.Libraries {
86+
for _, lib := range pipeline.Libraries {
10287
libs, err := m.expandLibrary(ctx, b.BundleRootPath, lib)
10388
if err != nil {
10489
return diag.FromErr(err)
10590
}
106-
10791
expanded = append(expanded, libs...)
108-
for range libs {
109-
sources = append(sources, []int{i})
110-
}
11192
}
112-
11393
pipeline.Libraries = expanded
114-
b.Config.UpdateSequence(path, sources)
11594
}
11695

11796
return nil

‎bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"github.com/databricks/cli/bundle/config/resources"
1616
"github.com/databricks/cli/bundle/internal/bundletest"
1717
"github.com/databricks/cli/libs/diag"
18-
"github.com/databricks/cli/libs/dyn"
1918
"github.com/databricks/databricks-sdk-go/service/compute"
2019
"github.com/databricks/databricks-sdk-go/service/pipelines"
2120
"github.com/stretchr/testify/require"
@@ -235,8 +234,8 @@ func TestExpandGlobPathsInPipelinesKeepsOrder(t *testing.T) {
235234
{File: &pipelines.FileLibrary{Path: "a/1.py"}},
236235
{File: &pipelines.FileLibrary{Path: "a/2.py"}},
237236
{File: &pipelines.FileLibrary{Path: "/Workspace/x.py"}},
238-
// The empty notebook is kept; the round trip through the dynamic configuration forces it.
239-
{Notebook: &pipelines.NotebookLibrary{}, ForceSendFields: []string{"Notebook"}},
237+
// The empty notebook is kept.
238+
{Notebook: &pipelines.NotebookLibrary{}},
240239
}, b.Config.Resources.Pipelines["pipeline"].Libraries)
241240

242241
// The original library is not modified.
@@ -265,8 +264,4 @@ func TestExpandGlobPathsInPipelinesWithoutLibraries(t *testing.T) {
265264

266265
assert.Nil(t, b.Config.Resources.Pipelines["unset"].Libraries)
267266
assert.Equal(t, []pipelines.PipelineLibrary{}, b.Config.Resources.Pipelines["empty"].Libraries)
268-
269-
v := b.Config.Value()
270-
assert.False(t, v.Get("resources").Get("pipelines").Get("unset").Get("libraries").IsValid())
271-
assert.Equal(t, dyn.KindSequence, v.Get("resources").Get("pipelines").Get("empty").Get("libraries").Kind())
272267
}

‎bundle/config/mutator/resourcemutator/process_static_resources_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,12 @@ package resourcemutator
22

33
import (
44
"reflect"
5+
"strings"
56
"testing"
67

78
"github.com/databricks/cli/bundle"
89
"github.com/databricks/cli/bundle/config"
910
"github.com/databricks/cli/bundle/config/resources"
10-
"github.com/databricks/cli/libs/dyn"
11-
"github.com/databricks/cli/libs/dyn/convert"
1211
"github.com/stretchr/testify/assert"
1312
"github.com/stretchr/testify/require"
1413
)
@@ -52,11 +51,12 @@ func TestGetAllResourcesMatchesConfigKeys(t *testing.T) {
5251
field.Set(m)
5352
}
5453

55-
root, err := convert.FromTyped(b.Config, dyn.NilValue)
56-
require.NoError(t, err)
54+
// The key of a resource type is the JSON name of its field in Resources.
5755
expected := NewResourceKeySet()
58-
err = expected.AddPattern(dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), root)
59-
require.NoError(t, err)
56+
for i := range rv.NumField() {
57+
typ, _, _ := strings.Cut(rv.Type().Field(i).Tag.Get("json"), ",")
58+
expected.AddResourceKey(ResourceKey{Type: typ, Name: "my_resource"})
59+
}
6060
require.Equal(t, rv.NumField(), expected.Size())
6161

6262
assert.Equal(t, expected, getAllResources(b))

0 commit comments

Comments
 (0)