Skip to content

Commit df4dadd

Browse files
denikIsaac
andauthored
diag: use structpath for diagnostic paths (#6980)
## Changes `diag.Diagnostic.Paths` is `[]*structpath.PathNode` instead of `[]dyn.Path`. Producers convert existing dyn paths with `dyn.ToStructPaths` or build paths from keys with `structpath.NewStringKeys`/`NewStringKeysSlice`/`MustParsePaths`. `structpath.PureReferenceToPath` moves to `dynvar.PureReferenceToStructPath` so structpath does not import dyn. User-visible: diagnostic paths quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']` instead of `resources.jobs.${var.env}_job` (one golden). ## Why First step of removing `libs/dyn` from bundle configuration: #6974 moves `Location` to `libs/diag` on top of this (it needs diag to no longer import dyn), then #6954. This pull request and its description were written by Isaac. --------- Co-authored-by: Isaac <no-reply@databricks.com>
1 parent a80cc8d commit df4dadd

56 files changed

Lines changed: 490 additions & 282 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.nextchanges/bundles/6980.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* Diagnostic paths now quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']`. ([#6980](https://github.com/databricks/cli/pull/6980))

‎acceptance/bundle/variables/variable_in_resource_key/output.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
Warning: unknown field: ${var.env}
2-
at targets.dev.resources.jobs.${var.env}_job_2
2+
at targets.dev.resources.jobs['${var.env}_job_2']
33
in databricks.yml:21:11
44

55
Error: resource key "${var.env}_job" must not contain variable references
6-
at resources.jobs.${var.env}_job
6+
at resources.jobs['${var.env}_job']
77
in databricks.yml:12:7
88

99
Error: resource key "${var.env}_job_2" must not contain variable references
10-
at targets.dev.resources.jobs.${var.env}_job_2
10+
at targets.dev.resources.jobs['${var.env}_job_2']
1111
in databricks.yml:20:11
1212

1313

‎bundle/apps/validate.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/databricks/cli/bundle/config/resources"
1010
"github.com/databricks/cli/libs/diag"
1111
"github.com/databricks/cli/libs/dyn"
12+
"github.com/databricks/cli/libs/structs/structpath"
1213
"github.com/databricks/databricks-sdk-go/service/apps"
1314
)
1415

@@ -27,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
2728
Severity: diag.Error,
2829
Summary: "Missing app source code path or git source",
2930
Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key),
30-
Locations: b.Config.GetLocations("resources.apps." + key),
31+
Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
3132
})
3233
continue
3334
}
@@ -37,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
3738
Severity: diag.Error,
3839
Summary: "Both source_code_path and git_source fields are set",
3940
Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key),
40-
Locations: b.Config.GetLocations("resources.apps." + key),
41+
Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
4142
})
4243
continue
4344
}
@@ -47,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
4748
Severity: diag.Error,
4849
Summary: "Duplicate app source code path",
4950
Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]),
50-
Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
51+
Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")),
5152
})
5253
}
5354
usedSourceCodePaths[app.SourceCodePath] = key
@@ -154,7 +155,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
154155
continue
155156
}
156157

157-
appPath := "resources.apps." + appKey
158+
appPath := structpath.NewPath(nil, "resources", "apps", appKey)
158159
diags = append(diags, diag.Diagnostic{
159160
Severity: diag.Warning,
160161
Summary: fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType),
@@ -172,8 +173,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
172173
ref.permission,
173174
appKey,
174175
),
175-
Paths: []dyn.Path{dyn.MustPathFromString(appPath)},
176-
Locations: b.Config.GetLocations(appPath),
176+
Paths: []*structpath.PathNode{appPath},
177+
Locations: b.Config.GetLocationsOf(appPath),
177178
})
178179
}
179180

‎bundle/apps/validate_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ func TestAppsValidateResourcePermissionsWarning(t *testing.T) {
213213
require.Len(t, warnings, 1)
214214
require.Contains(t, warnings[0].Summary, tc.wantSummary)
215215
require.Contains(t, warnings[0].Detail, "service_principal_name: ${resources.apps.my_app.service_principal_client_id}")
216-
require.Equal(t, dyn.MustPathFromString("resources.apps.my_app"), warnings[0].Paths[0])
216+
require.Equal(t, "resources.apps.my_app", warnings[0].Paths[0].String())
217217
} else {
218218
require.Empty(t, warnings)
219219
}

‎bundle/artifacts/expand_globs.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic {
2727
Severity: diag.Error,
2828
Summary: fmt.Sprintf("%s: %s", source, message),
2929
Locations: []dyn.Location{v.Location()},
30-
Paths: []dyn.Path{p},
30+
Paths: dyn.ToStructPaths(p),
3131
}
3232
}
3333

‎bundle/config/loader/process_include.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/databricks/cli/bundle/config"
1212
"github.com/databricks/cli/libs/diag"
1313
"github.com/databricks/cli/libs/dyn"
14+
"github.com/databricks/cli/libs/structs/structpath"
1415
)
1516

1617
func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics {
@@ -108,16 +109,16 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D
108109
}
109110

110111
var locations []dyn.Location
111-
var paths []dyn.Path
112+
var paths []*structpath.PathNode
112113
for _, rr := range resources {
113114
locations = append(locations, rr.value.Locations()...)
114-
paths = append(paths, rr.path)
115+
paths = append(paths, dyn.ToStructPath(rr.path))
115116
}
116117
// Sort the locations and paths to make the output deterministic.
117118
slices.SortFunc(locations, func(a, b dyn.Location) int {
118119
return cmp.Compare(a.String(), b.String())
119120
})
120-
slices.SortFunc(paths, func(a, b dyn.Path) int {
121+
slices.SortFunc(paths, func(a, b *structpath.PathNode) int {
121122
return cmp.Compare(a.String(), b.String())
122123
})
123124

@@ -168,7 +169,7 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
168169
Detail: `An include section is defined in a file that is not databricks.yml.
169170
Only includes defined in databricks.yml are applied.`,
170171
Locations: this.GetLocations("include"),
171-
Paths: []dyn.Path{dyn.MustPathFromString("include")},
172+
Paths: structpath.NewPathSlice("include"),
172173
})
173174
}
174175

‎bundle/config/loader/process_include_test.go‎

Lines changed: 38 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/databricks/cli/bundle/config/loader"
1010
"github.com/databricks/cli/libs/diag"
1111
"github.com/databricks/cli/libs/dyn"
12+
"github.com/databricks/cli/libs/structs/structpath"
1213
"github.com/stretchr/testify/assert"
1314
"github.com/stretchr/testify/require"
1415
)
@@ -71,10 +72,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
7172
{File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 11, Column: 11},
7273
{File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 4, Column: 7},
7374
},
74-
Paths: []dyn.Path{
75-
dyn.MustPathFromString("resources.jobs.job1"),
76-
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
77-
},
75+
Paths: structpath.MustParsePaths(
76+
"resources.jobs.job1",
77+
"targets.target1.resources.jobs.job1",
78+
),
7879
},
7980
},
8081
"job_and_pipeline.job.yml": {
@@ -86,10 +87,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
8687
{File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 11, Column: 11},
8788
{File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 4, Column: 7},
8889
},
89-
Paths: []dyn.Path{
90-
dyn.MustPathFromString("resources.pipelines.pipeline1"),
91-
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
92-
},
90+
Paths: structpath.MustParsePaths(
91+
"resources.pipelines.pipeline1",
92+
"targets.target1.resources.jobs.job1",
93+
),
9394
},
9495
},
9596
"job_and_pipeline.experiment.yml": {
@@ -101,10 +102,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
101102
{File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 11, Column: 11},
102103
{File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 4, Column: 7},
103104
},
104-
Paths: []dyn.Path{
105-
dyn.MustPathFromString("resources.pipelines.pipeline1"),
106-
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
107-
},
105+
Paths: structpath.MustParsePaths(
106+
"resources.pipelines.pipeline1",
107+
"targets.target1.resources.jobs.job1",
108+
),
108109
},
109110
},
110111
"two_jobs.job.yml": {
@@ -116,10 +117,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
116117
{File: "testdata/format_not_match/two_jobs.job.yml", Line: 4, Column: 7},
117118
{File: "testdata/format_not_match/two_jobs.job.yml", Line: 7, Column: 7},
118119
},
119-
Paths: []dyn.Path{
120-
dyn.MustPathFromString("resources.jobs.job1"),
121-
dyn.MustPathFromString("resources.jobs.job2"),
122-
},
120+
Paths: structpath.MustParsePaths(
121+
"resources.jobs.job1",
122+
"resources.jobs.job2",
123+
),
123124
},
124125
},
125126
"second_job_in_target.job.yml": {
@@ -131,10 +132,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
131132
{File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 11, Column: 11},
132133
{File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 4, Column: 7},
133134
},
134-
Paths: []dyn.Path{
135-
dyn.MustPathFromString("resources.jobs.job1"),
136-
dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
137-
},
135+
Paths: structpath.MustParsePaths(
136+
"resources.jobs.job1",
137+
"targets.target1.resources.jobs.job2",
138+
),
138139
},
139140
},
140141
"two_jobs_in_target.job.yml": {
@@ -146,10 +147,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
146147
{File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 6, Column: 11},
147148
{File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 8, Column: 11},
148149
},
149-
Paths: []dyn.Path{
150-
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
151-
dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
152-
},
150+
Paths: structpath.MustParsePaths(
151+
"targets.target1.resources.jobs.job1",
152+
"targets.target1.resources.jobs.job2",
153+
),
153154
},
154155
},
155156
"multiple_resources.model_serving_endpoint.yml": {
@@ -182,19 +183,19 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
182183
{File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 4, Column: 7},
183184
{File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 8, Column: 7},
184185
},
185-
Paths: []dyn.Path{
186-
dyn.MustPathFromString("resources.experiments.experiment1"),
187-
dyn.MustPathFromString("resources.jobs.job1"),
188-
dyn.MustPathFromString("resources.jobs.job2"),
189-
dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"),
190-
dyn.MustPathFromString("resources.models.model1"),
191-
dyn.MustPathFromString("resources.pipelines.pipeline1"),
192-
dyn.MustPathFromString("resources.pipelines.pipeline2"),
193-
dyn.MustPathFromString("resources.schemas.schema1"),
194-
dyn.MustPathFromString("targets.target1.resources.jobs.job3"),
195-
dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"),
196-
dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"),
197-
},
186+
Paths: structpath.MustParsePaths(
187+
"resources.experiments.experiment1",
188+
"resources.jobs.job1",
189+
"resources.jobs.job2",
190+
"resources.model_serving_endpoints.model_serving_endpoint1",
191+
"resources.models.model1",
192+
"resources.pipelines.pipeline1",
193+
"resources.pipelines.pipeline2",
194+
"resources.schemas.schema1",
195+
"targets.target1.resources.jobs.job3",
196+
"targets.target1.resources.quality_monitors.quality_monitor1",
197+
"targets.target1.resources.registered_models.registered_model1",
198+
),
198199
},
199200
},
200201
} {

‎bundle/config/mutator/apply_source_linked_deployment_preset.go‎

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/databricks/cli/bundle/config"
99
"github.com/databricks/cli/libs/dbr"
1010
"github.com/databricks/cli/libs/diag"
11-
"github.com/databricks/cli/libs/dyn"
11+
"github.com/databricks/cli/libs/structs/structpath"
1212
)
1313

1414
type applySourceLinkedDeploymentPreset struct{}
@@ -33,15 +33,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
3333

3434
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
3535
if !isDatabricksWorkspace {
36-
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
36+
path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment")
3737
diags = diags.Append(
3838
diag.Diagnostic{
39-
Severity: diag.Warning,
40-
Summary: "source-linked deployment is available only in the Databricks Workspace",
41-
Paths: []dyn.Path{
42-
path,
43-
},
44-
Locations: b.Config.GetLocations(path[2:].String()),
39+
Severity: diag.Warning,
40+
Summary: "source-linked deployment is available only in the Databricks Workspace",
41+
Paths: []*structpath.PathNode{path},
42+
Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)),
4543
},
4644
)
4745

@@ -53,15 +51,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
5351
b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development)
5452

5553
if b.Config.Bundle.Mode != config.Development {
56-
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
54+
path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment")
5755
diags = diags.Append(
5856
diag.Diagnostic{
59-
Severity: diag.Warning,
60-
Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release",
61-
Paths: []dyn.Path{
62-
path,
63-
},
64-
Locations: b.Config.GetLocations(path[2:].String()),
57+
Severity: diag.Warning,
58+
Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release",
59+
Paths: []*structpath.PathNode{path},
60+
Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)),
6561
},
6662
)
6763
}
@@ -74,16 +70,14 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
7470

7571
// This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value
7672
if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
77-
path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path"))
73+
path := structpath.NewPath(nil, "workspace", "file_path")
7874
diags = diags.Append(
7975
diag.Diagnostic{
80-
Severity: diag.Warning,
81-
Summary: "workspace.file_path setting will be ignored in source-linked deployment mode",
82-
Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead",
83-
Paths: []dyn.Path{
84-
path,
85-
},
86-
Locations: b.Config.GetLocations(path.String()),
76+
Severity: diag.Warning,
77+
Summary: "workspace.file_path setting will be ignored in source-linked deployment mode",
78+
Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead",
79+
Paths: []*structpath.PathNode{path},
80+
Locations: b.Config.GetLocationsOf(path),
8781
},
8882
)
8983
}

‎bundle/config/mutator/compute_id_compat.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn
5555
Severity: diag.Warning,
5656
Summary: "compute_id is deprecated, please use cluster_id instead",
5757
Locations: computeId.Locations(),
58-
Paths: []dyn.Path{computeIdPath},
58+
Paths: dyn.ToStructPaths(computeIdPath),
5959
})
6060

6161
clusterIdPath := p.Append(dyn.Key("cluster_id"))

‎bundle/config/mutator/load_dbalert_files.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/databricks/cli/libs/diag"
1414
"github.com/databricks/cli/libs/dyn"
1515
"github.com/databricks/cli/libs/dyn/dynvar"
16+
"github.com/databricks/cli/libs/structs/structpath"
1617
"github.com/databricks/databricks-sdk-go/marshal"
1718
"github.com/databricks/databricks-sdk-go/service/sql"
1819
)
@@ -85,7 +86,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
8586
Severity: diag.Error,
8687
Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k),
8788
Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "),
88-
Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))},
89+
Paths: structpath.NewPathSlice("resources", "alerts", alertKey, k),
8990
Locations: v.Locations(),
9091
},
9192
}
@@ -107,7 +108,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
107108
Severity: diag.Error,
108109
Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err),
109110
Detail: "",
110-
Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))},
111+
Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"),
111112
Locations: alertV.Get("file_path").Locations(),
112113
},
113114
}
@@ -122,7 +123,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
122123
Severity: diag.Error,
123124
Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err),
124125
Detail: "",
125-
Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))},
126+
Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"),
126127
Locations: alertV.Get("file_path").Locations(),
127128
},
128129
}
@@ -136,7 +137,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
136137
Severity: diag.Error,
137138
Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath),
138139
Detail: "Please inline the alert configuration in the bundle configuration to use variables",
139-
Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))},
140+
Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"),
140141
Locations: alertV.Get("file_path").Locations(),
141142
},
142143
}

0 commit comments

Comments
 (0)