|
1 | 1 | package paths |
2 | 2 |
|
3 | 3 | import ( |
| 4 | + "github.com/databricks/cli/bundle/config" |
4 | 5 | "github.com/databricks/cli/bundle/libraries" |
5 | 6 | "github.com/databricks/cli/libs/structs/structpath" |
6 | | - "github.com/databricks/cli/libs/structs/structvar" |
| 7 | + "github.com/databricks/databricks-sdk-go/service/jobs" |
7 | 8 | ) |
8 | 9 |
|
9 | | -func jobTaskLibrariesRewritePatterns(base *structpath.PatternNode) []jobRewritePattern { |
10 | | - return []jobRewritePattern{ |
11 | | - { |
12 | | - structpath.NewPattern(base, "libraries", structpath.AnyIndex, "whl"), |
13 | | - TranslateModeLocalRelative, |
14 | | - noSkipRewrite, |
15 | | - }, |
16 | | - { |
17 | | - structpath.NewPattern(base, "libraries", structpath.AnyIndex, "jar"), |
18 | | - TranslateModeLocalRelative, |
19 | | - noSkipRewrite, |
20 | | - }, |
21 | | - } |
| 10 | +func libraryRule(key string, field func(*jobs.Task, int) *string) taskRule { |
| 11 | + return taskRule{TranslateModeLocalRelative, nil, func(t *jobs.Task, c *taskCursor) error { |
| 12 | + for i := range t.Libraries { |
| 13 | + if err := c.yield(field(t, i), "libraries", i, key); err != nil { |
| 14 | + return err |
| 15 | + } |
| 16 | + } |
| 17 | + return nil |
| 18 | + }} |
22 | 19 | } |
23 | 20 |
|
24 | | -func jobLibrariesRewritePatterns() []jobRewritePattern { |
25 | | - // Base pattern to match all tasks in all jobs. |
26 | | - base := jobTasksPattern |
27 | | - |
28 | | - // Compile list of patterns and their respective rewrite functions. |
29 | | - jobEnvironmentsPatterns := []jobRewritePattern{ |
30 | | - { |
31 | | - structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies", structpath.AnyIndex), |
32 | | - TranslateModeLocalRelativeWithPrefix, |
33 | | - func(s string) bool { |
34 | | - return !libraries.IsLibraryLocal(s) |
35 | | - }, |
36 | | - }, |
37 | | - } |
| 21 | +var jobTaskLibrariesRules = []taskRule{ |
| 22 | + libraryRule("whl", func(t *jobs.Task, i int) *string { return &t.Libraries[i].Whl }), |
| 23 | + libraryRule("jar", func(t *jobs.Task, i int) *string { return &t.Libraries[i].Jar }), |
| 24 | +} |
38 | 25 |
|
39 | | - jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{ |
40 | | - { |
41 | | - structpath.NewPattern(nil, "resources", "jobs", structpath.AnyKey, "environments", structpath.AnyIndex, "spec", "dependencies", structpath.AnyIndex), |
42 | | - TranslateModeEnvironmentPipFlag, |
43 | | - func(s string) bool { |
44 | | - _, _, ok := libraries.IsLocalPathInPipFlag(s) |
45 | | - return !ok |
46 | | - }, |
47 | | - }, |
48 | | - } |
| 26 | +func notLocalLibrary(s string) bool { |
| 27 | + return !libraries.IsLibraryLocal(s) |
| 28 | +} |
49 | 29 |
|
50 | | - taskPatterns := jobTaskLibrariesRewritePatterns(base) |
51 | | - forEachPatterns := jobTaskLibrariesRewritePatterns(structpath.NewPattern(base, "for_each_task", "task")) |
52 | | - allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) |
53 | | - allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...) |
54 | | - allPatterns = append(allPatterns, forEachPatterns...) |
55 | | - return allPatterns |
| 30 | +func noLocalPathInPipFlag(s string) bool { |
| 31 | + _, _, ok := libraries.IsLocalPathInPipFlag(s) |
| 32 | + return !ok |
56 | 33 | } |
57 | 34 |
|
58 | 35 | // VisitJobLibrariesPaths visits all libraries related paths in job resources and applies a function to each path. |
59 | | -func VisitJobLibrariesPaths(root structvar.View, fn VisitFunc) error { |
60 | | - for _, rewritePattern := range jobLibrariesRewritePatterns() { |
61 | | - err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) |
62 | | - if err != nil { |
63 | | - return err |
| 36 | +func VisitJobLibrariesPaths(root *config.Root, fn VisitFunc) error { |
| 37 | + // Task libraries first, then environment dependencies (local libraries, then |
| 38 | + // local paths in pip flags), then the libraries of tasks nested in for_each_task. |
| 39 | + if err := visitTasksAt(root, false, jobTaskLibrariesRules, fn); err != nil { |
| 40 | + return err |
| 41 | + } |
| 42 | + for _, rule := range []struct { |
| 43 | + mode TranslateMode |
| 44 | + skip func(string) bool |
| 45 | + }{ |
| 46 | + {TranslateModeLocalRelativeWithPrefix, notLocalLibrary}, |
| 47 | + {TranslateModeEnvironmentPipFlag, noLocalPathInPipFlag}, |
| 48 | + } { |
| 49 | + for _, name := range sortedKeys(root.Resources.Jobs) { |
| 50 | + job := root.Resources.Jobs[name] |
| 51 | + if job == nil { |
| 52 | + continue |
| 53 | + } |
| 54 | + for i, env := range job.Environments { |
| 55 | + if env.Spec == nil { |
| 56 | + continue |
| 57 | + } |
| 58 | + for j := range env.Spec.Dependencies { |
| 59 | + if err := visit(&env.Spec.Dependencies[j], rule.mode, rule.skip, fn, func() *structpath.PathNode { |
| 60 | + return structpath.NewPath(resourcePath("jobs", name), "environments", i, "spec", "dependencies", j) |
| 61 | + }); err != nil { |
| 62 | + return err |
| 63 | + } |
| 64 | + } |
| 65 | + } |
64 | 66 | } |
65 | 67 | } |
66 | | - |
67 | | - return nil |
| 68 | + return visitTasksAt(root, true, jobTaskLibrariesRules, fn) |
68 | 69 | } |
0 commit comments