Skip to content

Commit e038a13

Browse files
tharunbhuk-dbtharunbhuk-db
committed
Keep the existing pipeline path rules and reads unchanged, adding the testing ones beside them
1 parent 742ef87 commit e038a13

3 files changed

Lines changed: 88 additions & 61 deletions

File tree

‎bundle/config/mutator/paths/pipeline_paths_visitor.go‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -55,31 +55,56 @@ func pipelineRewritePatterns() []pipelineRewritePattern {
5555
}
5656

5757
func pipelineLibrariesRewritePatterns() []pipelineRewritePattern {
58-
var patterns []pipelineRewritePattern
59-
// Test dependencies take the same forms as the pipeline's environment dependencies.
60-
for _, dependencies := range []dyn.Pattern{
61-
base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
62-
base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
63-
} {
64-
patterns = append(patterns,
65-
pipelineRewritePattern{
66-
pattern: dependencies,
67-
mode: TranslateModeLocalRelativeWithPrefix,
68-
skipRewrite: func(s string) bool {
69-
return !libraries.IsLibraryLocal(s)
70-
},
58+
pipelineEnvironmentsPatterns := []pipelineRewritePattern{
59+
{
60+
pattern: dyn.NewPattern(
61+
dyn.Key("resources"),
62+
dyn.Key("pipelines"),
63+
dyn.AnyKey(),
64+
dyn.Key("environment"),
65+
dyn.Key("dependencies"),
66+
dyn.AnyIndex(),
67+
),
68+
mode: TranslateModeLocalRelativeWithPrefix,
69+
skipRewrite: func(s string) bool {
70+
return !libraries.IsLibraryLocal(s)
71+
},
72+
},
73+
// Test dependencies take the same forms as the pipeline's environment dependencies.
74+
{
75+
pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
76+
mode: TranslateModeLocalRelativeWithPrefix,
77+
skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) },
78+
},
79+
}
80+
81+
pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{
82+
{
83+
dyn.NewPattern(
84+
dyn.Key("resources"),
85+
dyn.Key("pipelines"),
86+
dyn.AnyKey(),
87+
dyn.Key("environment"),
88+
dyn.Key("dependencies"),
89+
dyn.AnyIndex(),
90+
),
91+
TranslateModeEnvironmentPipFlag,
92+
func(s string) bool {
93+
_, _, ok := libraries.IsLocalPathInPipFlag(s)
94+
return !ok
7195
},
72-
pipelineRewritePattern{
73-
pattern: dependencies,
74-
mode: TranslateModeEnvironmentPipFlag,
75-
skipRewrite: func(s string) bool {
76-
_, _, ok := libraries.IsLocalPathInPipFlag(s)
77-
return !ok
78-
},
96+
},
97+
{
98+
base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
99+
TranslateModeEnvironmentPipFlag,
100+
func(s string) bool {
101+
_, _, ok := libraries.IsLocalPathInPipFlag(s)
102+
return !ok
79103
},
80-
)
104+
},
81105
}
82-
return patterns
106+
107+
return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...)
83108
}
84109

85110
func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {

‎bundle/config/mutator/translate_paths.go‎

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,15 @@ func (t *translateContext) rewritePath(
163163
localPath = filepath.ToSlash(localPath)
164164
localRelPath = filepath.ToSlash(localRelPath)
165165

166+
// With pipeline testing enabled, the path after -e can also be a project folder, with optional
167+
// extras such as ./project[test].
168+
if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" && env.ExperimentalPipelineTesting(ctx) {
169+
if interp, ok := t.translateEditableFolder(localRelPath); ok {
170+
t.seen[key] = interp
171+
return interp, nil
172+
}
173+
}
174+
166175
// Convert local path into workspace path via specified function.
167176
var interp string
168177
switch opts.Mode {
@@ -181,7 +190,9 @@ func (t *translateContext) rewritePath(
181190
case paths.TranslateModeLocalRelativeWithPrefix:
182191
interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath)
183192
case paths.TranslateModeEnvironmentPipFlag:
184-
interp, err = t.translatePipFlagPath(ctx, flag, input, localPath, localRelPath)
193+
interp, err = t.translateFilePath(ctx, input, localPath, localRelPath)
194+
// Add the flag prefix to the path to indicate it's a local file used in pip flags for environment dependencies.
195+
interp = flag + " " + interp
185196
default:
186197
return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode)
187198
}
@@ -250,24 +261,18 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local
250261
return path.Join(t.remoteRoot, localRelPath), nil
251262
}
252263

253-
// translatePipFlagPath translates the local path after a pip flag of an environment dependency,
254-
// and keeps the flag in front of it. With pipeline testing enabled, the path after -e can also be
255-
// a project folder, with optional extras such as ./project[test].
256-
func (t *translateContext) translatePipFlagPath(ctx context.Context, flag, literal, localFullPath, localRelPath string) (string, error) {
257-
if flag == "-e" && env.ExperimentalPipelineTesting(ctx) {
258-
folder, extras := localRelPath, ""
259-
if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") {
260-
folder, extras = localRelPath[:i], localRelPath[i:]
261-
}
262-
if info, err := t.b.SyncRoot.Stat(folder); err == nil && info.IsDir() {
263-
return flag + " " + path.Join(t.remoteRoot, folder) + extras, nil
264-
}
264+
// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to
265+
// its deployed folder. It reports false if the path is not a folder in the sync root.
266+
func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) {
267+
folder, extras := localRelPath, ""
268+
if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") {
269+
folder, extras = localRelPath[:i], localRelPath[i:]
265270
}
266-
interp, err := t.translateFilePath(ctx, literal, localFullPath, localRelPath)
267-
if err != nil {
268-
return "", err
271+
info, err := t.b.SyncRoot.Stat(folder)
272+
if err != nil || !info.IsDir() {
273+
return "", false
269274
}
270-
return flag + " " + interp, nil
275+
return "-e " + path.Join(t.remoteRoot, folder) + extras, true
271276
}
272277

273278
func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {

‎bundle/direct/dresources/pipeline.go‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -78,20 +78,12 @@ func (s PipelineRemote) MarshalJSON() ([]byte, error) {
7878

7979
type ResourcePipeline struct {
8080
client *databricks.WorkspaceClient
81-
82-
// apiClient sends the requests that the Go SDK cannot express yet because it lacks the testing
83-
// field. It is built once, so that those requests share one rate limiter.
84-
apiClient *client.DatabricksClient
85-
apiClientErr error
8681
}
8782

88-
func (*ResourcePipeline) New(w *databricks.WorkspaceClient) *ResourcePipeline {
89-
r := &ResourcePipeline{client: w, apiClient: nil, apiClientErr: nil}
90-
// The adapter also builds a resource without a client to read its configuration.
91-
if w != nil {
92-
r.apiClient, r.apiClientErr = client.New(w.Config)
83+
func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeline {
84+
return &ResourcePipeline{
85+
client: client,
9386
}
94-
return r
9587
}
9688

9789
func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState {
@@ -118,16 +110,20 @@ type pipelineTestingResponse struct {
118110
} `json:"spec"`
119111
}
120112

121-
// DoRead sends the same request as GetByPipelineId, but raw, so that it can also read spec.testing,
122-
// which GetPipelineResponse does not have yet. Switch back to GetByPipelineId once it does.
123113
func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) {
124-
if !env.ExperimentalPipelineTesting(ctx) {
125-
resp, err := r.client.Pipelines.GetByPipelineId(ctx, id)
126-
if err != nil {
127-
return nil, err
128-
}
129-
return makePipelineRemote(resp), nil
114+
if env.ExperimentalPipelineTesting(ctx) {
115+
return r.doReadWithTesting(ctx, id)
130116
}
117+
resp, err := r.client.Pipelines.GetByPipelineId(ctx, id)
118+
if err != nil {
119+
return nil, err
120+
}
121+
return makePipelineRemote(resp), nil
122+
}
123+
124+
// doReadWithTesting sends the same request as GetByPipelineId, but raw, so that it can also read
125+
// spec.testing, which GetPipelineResponse does not have yet. Remove it once GetPipelineResponse has it.
126+
func (r *ResourcePipeline) doReadWithTesting(ctx context.Context, id string) (*PipelineRemote, error) {
131127
var raw json.RawMessage
132128
if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil {
133129
return nil, err
@@ -325,10 +321,11 @@ func (r *ResourcePipeline) doWithTesting(ctx context.Context, method, path strin
325321

326322
// doRaw sends a request that the Go SDK cannot express yet because it lacks the testing field.
327323
func (r *ResourcePipeline) doRaw(ctx context.Context, method, path string, request, response any) error {
328-
if r.apiClientErr != nil {
329-
return fmt.Errorf("failed to create API client: %w", r.apiClientErr)
324+
apiClient, err := client.New(r.client.Config)
325+
if err != nil {
326+
return fmt.Errorf("failed to create API client: %w", err)
330327
}
331-
return r.apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response)
328+
return apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response)
332329
}
333330

334331
// Note, terraform provider either

0 commit comments

Comments
 (0)