Skip to content

Commit 42ec6ff

Browse files
denikIsaac
andcommitted
paths: typed visitors instead of pattern traversal
The path visitors used by TranslatePaths and NormalizePaths iterate the typed resources with plain field access and pass the value as a *string, so the consumers rewrite it in place instead of looking the path up again with Set. Rules are applied one at a time over sorted resource keys, as the patterns were, so the visit order is unchanged. Paths are only built for the values that are visited. 1000 jobs (main / #6954 / this): TranslatePaths 384 / 28 / 8 ms, NormalizePaths 395 / 50 / 26 ms. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 0b84018 commit 42ec6ff

19 files changed

Lines changed: 415 additions & 330 deletions

‎bundle/config/mutator/normalize_paths.go‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"github.com/databricks/cli/bundle/libraries"
1616
"github.com/databricks/cli/libs/diag"
1717
"github.com/databricks/cli/libs/structs/structpath"
18-
"github.com/databricks/cli/libs/structs/structvar"
1918
)
2019

2120
type normalizePaths struct{}
@@ -44,22 +43,18 @@ func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnost
4443
// Do not normalize job task paths if using git source
4544
gitSourcePaths := collectGitSourcePaths(b)
4645

47-
err := paths.VisitPaths(b.Config.View(), func(path *structpath.PathNode, kind paths.TranslateMode, v structvar.View) error {
46+
err := paths.VisitPaths(&b.Config, func(path *structpath.PathNode, kind paths.TranslateMode, s *string) error {
4847
if slices.ContainsFunc(gitSourcePaths, path.HasPrefix) {
4948
return nil
5049
}
5150

52-
value, ok := v.AsString()
53-
if !ok {
54-
return fmt.Errorf("value at %s is not a string", path.String())
55-
}
56-
57-
newValue, err := normalizePath(value, v.Location(), b.BundleRootPath)
51+
newValue, err := normalizePath(*s, b.Config.ExactLocation(path), b.BundleRootPath)
5852
if err != nil {
5953
return err
6054
}
6155

62-
return b.Config.Set(path, newValue)
56+
*s = newValue
57+
return nil
6358
})
6459
if err != nil {
6560
return diag.FromErr(fmt.Errorf("failed to normalize paths: %w", err))
Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package paths
22

33
import (
4+
"github.com/databricks/cli/bundle/config"
45
"github.com/databricks/cli/libs/structs/structpath"
5-
"github.com/databricks/cli/libs/structs/structvar"
66
)
77

8-
func VisitAlertPaths(root structvar.View, fn VisitFunc) error {
9-
pattern := structpath.NewPattern(nil, "resources", "alerts", structpath.AnyKey, "file_path")
10-
11-
return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error {
12-
return fn(path, TranslateModeLocalRelative, value)
13-
})
8+
func VisitAlertPaths(root *config.Root, fn VisitFunc) error {
9+
for _, name := range sortedKeys(root.Resources.Alerts) {
10+
if a := root.Resources.Alerts[name]; a != nil {
11+
if err := visit(&a.FilePath, TranslateModeLocalRelative, nil, fn, func() *structpath.PathNode { return structpath.NewPath(resourcePath("alerts", name), "file_path") }); err != nil {
12+
return err
13+
}
14+
}
15+
}
16+
return nil
1417
}
Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package paths
22

33
import (
4+
"github.com/databricks/cli/bundle/config"
45
"github.com/databricks/cli/libs/structs/structpath"
5-
"github.com/databricks/cli/libs/structs/structvar"
66
)
77

8-
func VisitAppPaths(root structvar.View, fn VisitFunc) error {
9-
pattern := structpath.NewPattern(nil, "resources", "apps", structpath.AnyKey, "source_code_path")
10-
11-
return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error {
12-
return fn(path, TranslateModeDirectory, value)
13-
})
8+
func VisitAppPaths(root *config.Root, fn VisitFunc) error {
9+
for _, name := range sortedKeys(root.Resources.Apps) {
10+
if a := root.Resources.Apps[name]; a != nil {
11+
if err := visit(&a.SourceCodePath, TranslateModeDirectory, nil, fn, func() *structpath.PathNode { return structpath.NewPath(resourcePath("apps", name), "source_code_path") }); err != nil {
12+
return err
13+
}
14+
}
15+
}
16+
return nil
1417
}
Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,17 @@
11
package paths
22

33
import (
4+
"github.com/databricks/cli/bundle/config"
45
"github.com/databricks/cli/libs/structs/structpath"
5-
"github.com/databricks/cli/libs/structs/structvar"
66
)
77

8-
type artifactRewritePattern struct {
9-
pattern *structpath.PatternNode
10-
mode TranslateMode
11-
}
12-
13-
func artifactRewritePatterns() []artifactRewritePattern {
14-
// Base pattern to match all artifacts.
15-
base := structpath.NewPattern(nil, "artifacts", structpath.AnyKey)
16-
17-
// Compile list of configuration paths to rewrite.
18-
return []artifactRewritePattern{
19-
{
20-
pattern: structpath.NewPattern(base, "path"),
21-
mode: TranslateModeLocalAbsoluteDirectory,
22-
},
23-
}
24-
}
25-
26-
func VisitArtifactPaths(root structvar.View, fn VisitFunc) error {
27-
for _, rewritePattern := range artifactRewritePatterns() {
28-
err := structvar.ForEach(root, rewritePattern.pattern, func(p *structpath.PathNode, v structvar.View) error {
29-
return fn(p, rewritePattern.mode, v)
30-
})
31-
if err != nil {
32-
return err
8+
func VisitArtifactPaths(root *config.Root, fn VisitFunc) error {
9+
for _, name := range sortedKeys(root.Artifacts) {
10+
if a := root.Artifacts[name]; a != nil {
11+
if err := visit(&a.Path, TranslateModeLocalAbsoluteDirectory, nil, fn, func() *structpath.PathNode { return structpath.NewPath(nil, "artifacts", name, "path") }); err != nil {
12+
return err
13+
}
3314
}
3415
}
35-
3616
return nil
3717
}
Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package paths
22

33
import (
4+
"github.com/databricks/cli/bundle/config"
45
"github.com/databricks/cli/libs/structs/structpath"
5-
"github.com/databricks/cli/libs/structs/structvar"
66
)
77

8-
func VisitDashboardPaths(root structvar.View, fn VisitFunc) error {
9-
pattern := structpath.NewPattern(nil, "resources", "dashboards", structpath.AnyKey, "file_path")
10-
11-
return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error {
12-
return fn(path, TranslateModeLocalRelative, value)
13-
})
8+
func VisitDashboardPaths(root *config.Root, fn VisitFunc) error {
9+
for _, name := range sortedKeys(root.Resources.Dashboards) {
10+
if d := root.Resources.Dashboards[name]; d != nil {
11+
if err := visit(&d.FilePath, TranslateModeLocalRelative, nil, fn, func() *structpath.PathNode { return structpath.NewPath(resourcePath("dashboards", name), "file_path") }); err != nil {
12+
return err
13+
}
14+
}
15+
}
16+
return nil
1417
}
Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
package paths
22

33
import (
4+
"github.com/databricks/cli/bundle/config"
45
"github.com/databricks/cli/libs/structs/structpath"
5-
"github.com/databricks/cli/libs/structs/structvar"
66
)
77

8-
func VisitGenieSpacePaths(root structvar.View, fn VisitFunc) error {
9-
pattern := structpath.NewPattern(nil, "resources", "genie_spaces", structpath.AnyKey, "file_path")
10-
11-
return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error {
12-
return fn(path, TranslateModeLocalRelative, value)
13-
})
8+
func VisitGenieSpacePaths(root *config.Root, fn VisitFunc) error {
9+
for _, name := range sortedKeys(root.Resources.GenieSpaces) {
10+
if g := root.Resources.GenieSpaces[name]; g != nil {
11+
if err := visit(&g.FilePath, TranslateModeLocalRelative, nil, fn, func() *structpath.PathNode {
12+
return structpath.NewPath(resourcePath("genie_spaces", name), "file_path")
13+
}); err != nil {
14+
return err
15+
}
16+
}
17+
}
18+
return nil
1419
}
Lines changed: 52 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,69 @@
11
package paths
22

33
import (
4+
"github.com/databricks/cli/bundle/config"
45
"github.com/databricks/cli/bundle/libraries"
56
"github.com/databricks/cli/libs/structs/structpath"
6-
"github.com/databricks/cli/libs/structs/structvar"
7+
"github.com/databricks/databricks-sdk-go/service/jobs"
78
)
89

9-
func jobTaskLibrariesRewritePatterns(base *structpath.PatternNode) []jobRewritePattern {
10-
return []jobRewritePattern{
11-
{
12-
structpath.NewPattern(base, "libraries", structpath.AnyIndex, "whl"),
13-
TranslateModeLocalRelative,
14-
noSkipRewrite,
15-
},
16-
{
17-
structpath.NewPattern(base, "libraries", structpath.AnyIndex, "jar"),
18-
TranslateModeLocalRelative,
19-
noSkipRewrite,
20-
},
21-
}
10+
func libraryRule(key string, field func(*jobs.Task, int) *string) taskRule {
11+
return taskRule{TranslateModeLocalRelative, nil, func(t *jobs.Task, c *taskCursor) error {
12+
for i := range t.Libraries {
13+
if err := c.yield(field(t, i), "libraries", i, key); err != nil {
14+
return err
15+
}
16+
}
17+
return nil
18+
}}
2219
}
2320

24-
func jobLibrariesRewritePatterns() []jobRewritePattern {
25-
// Base pattern to match all tasks in all jobs.
26-
base := jobTasksPattern
27-
28-
// Compile list of patterns and their respective rewrite functions.
29-
jobEnvironmentsPatterns := []jobRewritePattern{
30-
{
31-
structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies", structpath.AnyIndex),
32-
TranslateModeLocalRelativeWithPrefix,
33-
func(s string) bool {
34-
return !libraries.IsLibraryLocal(s)
35-
},
36-
},
37-
}
21+
var jobTaskLibrariesRules = []taskRule{
22+
libraryRule("whl", func(t *jobs.Task, i int) *string { return &t.Libraries[i].Whl }),
23+
libraryRule("jar", func(t *jobs.Task, i int) *string { return &t.Libraries[i].Jar }),
24+
}
3825

39-
jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{
40-
{
41-
structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies", structpath.AnyIndex),
42-
TranslateModeEnvironmentPipFlag,
43-
func(s string) bool {
44-
_, _, ok := libraries.IsLocalPathInPipFlag(s)
45-
return !ok
46-
},
47-
},
48-
}
26+
func notLocalLibrary(s string) bool {
27+
return !libraries.IsLibraryLocal(s)
28+
}
4929

50-
taskPatterns := jobTaskLibrariesRewritePatterns(base)
51-
forEachPatterns := jobTaskLibrariesRewritePatterns(structpath.NewPattern(base, "for_each_task", "task"))
52-
allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
53-
allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...)
54-
allPatterns = append(allPatterns, forEachPatterns...)
55-
return allPatterns
30+
func noLocalPathInPipFlag(s string) bool {
31+
_, _, ok := libraries.IsLocalPathInPipFlag(s)
32+
return !ok
5633
}
5734

5835
// VisitJobLibrariesPaths visits all libraries related paths in job resources and applies a function to each path.
59-
func VisitJobLibrariesPaths(root structvar.View, fn VisitFunc) error {
60-
for _, rewritePattern := range jobLibrariesRewritePatterns() {
61-
err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn)
62-
if err != nil {
63-
return err
36+
func VisitJobLibrariesPaths(root *config.Root, fn VisitFunc) error {
37+
// Task libraries first, then environment dependencies (local libraries, then
38+
// local paths in pip flags), then the libraries of tasks nested in for_each_task.
39+
if err := visitTasksAt(root, false, jobTaskLibrariesRules, fn); err != nil {
40+
return err
41+
}
42+
for _, rule := range []struct {
43+
mode TranslateMode
44+
skip func(string) bool
45+
}{
46+
{TranslateModeLocalRelativeWithPrefix, notLocalLibrary},
47+
{TranslateModeEnvironmentPipFlag, noLocalPathInPipFlag},
48+
} {
49+
for _, name := range sortedKeys(root.Resources.Jobs) {
50+
job := root.Resources.Jobs[name]
51+
if job == nil {
52+
continue
53+
}
54+
for i, env := range job.Environments {
55+
if env.Spec == nil {
56+
continue
57+
}
58+
for j := range env.Spec.Dependencies {
59+
if err := visit(&env.Spec.Dependencies[j], rule.mode, rule.skip, fn, func() *structpath.PathNode {
60+
return structpath.NewPath(resourcePath("jobs", name), "environments", i, "spec", "dependencies", j)
61+
}); err != nil {
62+
return err
63+
}
64+
}
65+
}
6466
}
6567
}
66-
67-
return nil
68+
return visitTasksAt(root, true, jobTaskLibrariesRules, fn)
6869
}

0 commit comments

Comments
 (0)