From 52dc912bf7cb4a0ea637b873fab539bc763641d3 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 11 Nov 2026 21:57:50 +0000 Subject: [PATCH] Translate the testing paths of pipelines and upload local test wheels Relative paths in testing.sources and testing.environment.dependencies resolve to their deployed workspace paths, like libraries and environment dependencies. Local wheels in test dependencies are uploaded like other libraries, and the same-name check covers them. The testing property is only accepted with DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true, so bundles without it are not changed. Co-authored-by: Isaac --- .../databricks.yml | 21 +++++++++++++++++++ .../testing-same-name-libraries/out.test.toml | 2 ++ .../testing-same-name-libraries/output.txt | 10 +++++++++ .../test_helpers-0.1-py3-none-any.whl | 0 .../testing-same-name-libraries/script | 1 + .../testing-same-name-libraries/test.toml | 7 +++++++ .../tests/test_helpers-0.1-py3-none-any.whl | 0 .../tests/test_orders.py | 2 ++ .../transformations/orders.py | 1 + .../dist/test_helpers-0.1-py3-none-any.whl | 0 .../resources/pipelines/testing/output.txt | 8 ++++--- .../bundle/resources/pipelines/testing/script | 1 + .../mutator/paths/pipeline_paths_visitor.go | 19 +++++++++++++++++ .../paths/pipeline_paths_visitor_test.go | 7 +++++++ bundle/libraries/expand_glob_references.go | 13 ++++++++++++ bundle/libraries/remote_path.go | 1 + bundle/libraries/same_name_libraries.go | 1 + 17 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/script create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..68399999172 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..32f10d10630 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d018f5851ee --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,7 @@ +# Runs only against the local test server. +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index 6133e93eb75..02708b42ed8 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -1,17 +1,19 @@ +=== Relative test paths resolve to the deployed workspace paths + >>> [CLI] bundle validate -o json { "environment": { "dependencies": [ - "-e .[test]", - "-r ./tests/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", "./dist/*.whl" ] }, "sources": [ { "glob": { - "include": "./tests/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" } } ] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index 1dc320485b7..ce456b64cbe 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1,3 +1,4 @@ +title "Relative test paths resolve to the deployed workspace paths" trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing title "Without pipeline testing, the property is an error" diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..83be40afd7d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns @@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !libraries.IsLibraryLocal(s) }, }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }, } pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ @@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !ok }, }, + { + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, + }, } return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..85c29065c5b 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) { }, }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &resources.PipelineTesting{ + Environment: &resources.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index ff6be05c3b2..020fd5bbdaa 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct {