Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
bundle:
name: pipeline-testing-same-name-libraries

resources:
pipelines:
orders:
name: orders
libraries:
- file:
path: ./transformations/orders.py
environment:
dependencies:
- ./pipeline/test_helpers-0.1-py3-none-any.whl
testing:
sources:
- glob:
include: ./tests/**
environment:
dependencies:
# A different wheel with the same file name would be uploaded to the same path.
- ./tests/test_helpers-0.1-py3-none-any.whl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@

>>> musterr [CLI] bundle deploy
Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl
at resources.pipelines.orders.environment.dependencies[0]
resources.pipelines.orders.testing.environment.dependencies[0]
in databricks.yml:13:13
databricks.yml:21:15

Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
trace musterr $CLI bundle deploy
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Runs only against the local test server.
Cloud = false
Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true"

[[Repls]]
Old = '\\'
New = '/'
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def test_orders():
pass
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
x = 1
8 changes: 5 additions & 3 deletions acceptance/bundle/resources/pipelines/testing/output.txt
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@

=== Relative test paths resolve to the deployed workspace paths

>>> [CLI] bundle validate -o json
{
"environment": {
"dependencies": [
"-e .[test]",
"-r ./tests/requirements.txt",
"-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]",
"-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt",
"./dist/*.whl"
]
},
"sources": [
{
"glob": {
"include": "./tests/**"
"include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**"
}
}
]
Expand Down
1 change: 1 addition & 0 deletions acceptance/bundle/resources/pipelines/testing/script
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
title "Relative test paths resolve to the deployed workspace paths"
trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a bundle deploy here (RecordRequests is already on in test.toml) so we see the test wheel uploaded and ./dist/*.whl rewritten to its .internal path? This test only runs validate, so the wheel stays ./dist/*.whl in the output, and the same-name test fails before upload, so nothing in the goldens shows the upload the description talks about. If that has to wait for #7438, worth a note in the script.


title "Without pipeline testing, the property is an error"
Expand Down
19 changes: 19 additions & 0 deletions bundle/config/mutator/paths/pipeline_paths_visitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern {
mode: TranslateModeDirectory,
skipRewrite: noSkipRewrite,
},
{
pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")),
mode: TranslateModeGlob,
skipRewrite: noSkipRewrite,
},
}

return allPatterns
Expand All @@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern {
return !libraries.IsLibraryLocal(s)
},
},
// Test dependencies take the same forms as the pipeline's environment dependencies.
{
pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the four dependency entries now differ only in the path prefix. Maybe loop over the two locations (environment and testing.environment) so the modes and skip functions live in one place?

mode: TranslateModeLocalRelativeWithPrefix,
skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) },
},
}

pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{
Expand All @@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern {
return !ok
},
},
{
base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
TranslateModeEnvironmentPipFlag,
func(s string) bool {
_, _, ok := libraries.IsLocalPathInPipFlag(s)
return !ok
},
},
}

return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...)
Expand Down
7 changes: 7 additions & 0 deletions bundle/config/mutator/paths/pipeline_paths_visitor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) {
},
},
},
// The acceptance test bundle/resources/pipelines/testing covers the testing paths
// that are rewritten. A package name is not a path.
Testing: &resources.PipelineTesting{
Environment: &resources.PipelineTestEnvironment{
Dependencies: []string{"faker"},
},
},
},
},
},
Expand Down
13 changes: 13 additions & 0 deletions bundle/libraries/expand_glob_references.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern(
dyn.Key("dependencies"),
)

var pipelineTestingEnvDepsPattern = dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("pipelines"),
dyn.AnyKey(),
dyn.Key("testing"),
dyn.Key("environment"),
dyn.Key("dependencies"),
)

var clusterLibrariesPattern = dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("clusters"),
Expand All @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
pattern: pipelineEnvDepsPattern,
fn: expandEnvironmentDeps,
},
{
pattern: pipelineTestingEnvDepsPattern,
fn: expandEnvironmentDeps,
},
{
pattern: clusterLibrariesPattern,
fn: expandLibraries,
Expand Down
1 change: 1 addition & 0 deletions bundle/libraries/remote_path.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err
clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")),
envDepsPattern.Append(dyn.AnyIndex()),
pipelineEnvDepsPattern.Append(dyn.AnyIndex()),
pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()),
// The AI Runtime task's code_source_path is a local archive (typically an
// artifact-built .tar.gz) that must be uploaded and referenced by its remote
// path, exactly like a wheel or jar library.
Expand Down
1 change: 1 addition & 0 deletions bundle/libraries/same_name_libraries.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{
clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")),
envDepsPattern.Append(dyn.AnyIndex()),
pipelineEnvDepsPattern.Append(dyn.AnyIndex()),
pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()),
}

type libData struct {
Expand Down
Loading