Run pipelines test against the pipeline's testing.sources

acceptance/pipelines/test/arguments/out.flag-error.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="unsupported pytest argument &#34;-x&#34;: only -k and -m are supported">unsupported pytest argument &#34;-x&#34;: only -k and -m are supported</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.gate-off.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it">pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.no-wait.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="--junit-xml cannot be used with --no-wait because no results are available when the command returns">--junit-xml cannot be used with --no-wait because no results are available when the command returns</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/output.txt

@@ -23,12 +23,30 @@ example:
 
   databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'
 
+The command waits for the results. It exits with code 0 only if at least one
+test was collected and none failed or errored, so a run whose tests were all
+skipped succeeds; otherwise it exits with code 1. --no-wait returns after the
+update starts.
+
+--timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume
+to read the results later. To stop the update, run 'pipelines stop'.
+
+--junit-xml writes one entry per test with its outcome and duration. An
+incomplete run, a run without tests, or a timeout adds a failing entry that
+names the cause. The report replaces the file on every exit except a flag that
+cannot be parsed. With workspace.sync_mode immutable, test names keep the path
+of the deployed snapshot, which changes with each deployment.
+
 Usage:
   databricks pipelines test [flags] [KEY] [TEST_SELECTION...]
 
 Pipeline Test Flags:
+      --junit-xml string         Write the results as a JUnit XML report to this file, replacing it.
+      --no-wait                  Don't wait for the tests to complete.
       --pipeline-id string       ID of the pipeline to test, instead of a KEY.
       --pytest-arg stringArray   Argument to pass to pytest (repeatable). Only -k and -m are supported.
+      --resume string            Read the results of an existing test update instead of starting one.
+      --timeout duration         Stop waiting for results after this duration, for example 30m. The update keeps running.
 
 Flags:
   -h, --help   help for test
@@ -45,6 +63,11 @@ Global Flags:
 >>> musterr [CLI] pipelines test --pipeline-id 1234
 Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
 
+=== With the command off, an older report is still replaced
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml
+Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
+
 === pytest arguments must be passed with --pytest-arg
 
 >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders
@@ -70,3 +93,33 @@ Error: --pipeline-id "1234" is not a pipeline ID
 
 >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json
 Error: --output json is not supported by pipelines test
+
+=== Incompatible run-control flags
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 tests --resume 5678
+Error: --resume cannot be used with test selections
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke
+Error: --resume cannot be used with --pytest-arg
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --no-wait
+Error: --resume cannot be used with --no-wait
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --timeout 10m
+Error: --timeout cannot be used with --no-wait
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678' OR '1'='1
+Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that pipelines test printed
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 
+Error: --resume "" is not an update ID; pass the update ID that pipelines test printed
+
+=== --junit-xml needs the results
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns
+
+=== An invalid flag replaces an older report
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml
+Error: unsupported pytest argument "-x": only -k and -m are supported

acceptance/pipelines/test/arguments/script

@@ -3,6 +3,10 @@ trace $CLI pipelines test --help
 title "The command is experimental and off by default"
 (unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234)
 
+title "With the command off, an older report is still replaced"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.gate-off.xml
+(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml)
+
 title "pytest arguments must be passed with --pytest-arg"
 trace musterr $CLI pipelines test --pipeline-id 1234 -k orders
 trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders
@@ -16,3 +20,18 @@ trace musterr $CLI pipelines test --pipeline-id 1234
 
 title "JSON output is not supported"
 trace musterr $CLI pipelines test --pipeline-id 1234 -o json
+
+title "Incompatible run-control flags"
+trace musterr $CLI pipelines test --pipeline-id 1234 tests --resume 5678
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait
+trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1"
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume ""
+
+title "--junit-xml needs the results"
+trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+
+title "An invalid flag replaces an older report"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.flag-error.xml
+trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml

acceptance/pipelines/test/immutable/databricks.yml

@@ -11,3 +11,7 @@ resources:
       libraries:
         - file:
             path: pipeline_file.py
+      testing:
+        sources:
+          - glob:
+              include: ./tests/**

acceptance/pipelines/test/immutable/output.txt

@@ -17,15 +17,15 @@ Error: cannot resolve test selection "tests/test_orders.py::test_backfill": loca
 
 === Absolute workspace paths and running all tests still work
 
->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill
+>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
 
 Pipeline: my_pipeline ([UUID][0])
 Update: [UUID][1]
 
->>> [CLI] pipelines test my_pipeline
+>>> [CLI] pipelines test my_pipeline --no-wait
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.

acceptance/pipelines/test/immutable/script

@@ -5,6 +5,6 @@ title "Local test paths are not supported with workspace.sync_mode immutable"
 trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill
 
 title "Absolute workspace paths and running all tests still work"
-trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill
-trace $CLI pipelines test my_pipeline
+trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait
+trace $CLI pipelines test my_pipeline --no-wait
 trace print_requests.py //updates

acceptance/pipelines/test/invalid-project/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-pipeline-test-invalid-project
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: ${var.missing}

acceptance/pipelines/test/invalid-project/out.invalid.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="reference does not exist: ${var.missing}">reference does not exist: ${var.missing}</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/invalid-project/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/pipelines/test/invalid-project/output.txt

@@ -0,0 +1,6 @@
+
+=== A project with errors names its first error in the report
+
+>>> musterr [CLI] pipelines test my_pipeline --junit-xml out.invalid.xml
+Error: reference does not exist: ${var.missing}
+

acceptance/pipelines/test/invalid-project/script

@@ -0,0 +1,2 @@
+title "A project with errors names its first error in the report"
+trace musterr $CLI pipelines test my_pipeline --junit-xml out.invalid.xml

acceptance/pipelines/test/live/databricks.yml.tmpl

@@ -0,0 +1,22 @@
+bundle:
+  name: test-pipelines-test-live-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    orders:
+      # The Pipelines API rejects duplicate names, so the name is unique per run.
+      name: test-pipelines-test-live-$UNIQUE_NAME
+      catalog: main
+      schema: default
+      serverless: true
+      root_path: .
+      libraries:
+        - file:
+            path: ./transformations/orders.py
+      testing:
+        sources:
+          - glob:
+              include: ./tests/**
+        environment:
+          dependencies:
+            - -e ./helpers

acceptance/pipelines/test/live/helpers/order_helpers.py

@@ -0,0 +1,2 @@
+def order_total(amounts):
+    return sum(amounts)

acceptance/pipelines/test/live/helpers/pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "order-helpers"
+version = "0.1.0"
+
+[tool.setuptools]
+py-modules = ["order_helpers"]

acceptance/pipelines/test/live/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+CloudSlow = true

acceptance/pipelines/test/live/output.txt

@@ -0,0 +1,22 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default/files...
+Created pipelines.orders
+Files: 6 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Run the tests and wait for the results
+Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=1
+ name="test_order_total"
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.orders
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.orders
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/pipelines/test/live/script

@@ -0,0 +1,20 @@
+if [ -n "$CLOUD_ENV" ] && [ -z "${TEST_PIPELINES_TESTING_ENABLED:-}" ]; then
+    echo "SKIP_TEST set TEST_PIPELINES_TESTING_ENABLED for a workspace with pipeline testing enabled"
+    exit 0
+fi
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+title "Run the tests and wait for the results"
+# Durations and test IDs differ between runs, so only the summary line and the case names are
+# compared. The script stops if the command fails.
+$CLI pipelines test --junit-xml junit.xml &> LOG.test
+tail -1 LOG.test
+grep -o ' name="test_[^"]*"' junit.xml

acceptance/pipelines/test/live/test.toml

@@ -0,0 +1,20 @@
+# Also runs against a real workspace, when that workspace has pipeline testing enabled.
+# The script skips the cloud run unless TEST_PIPELINES_TESTING_ENABLED is set.
+Cloud = true
+CloudSlow = true
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "junit.xml"]
+
+# Locally, the test server stands in for the service's test run. Start Update is the test server's.
+[[Server]]
+Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}"
+Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "COMPLETED", "test_only": true}}'
+
+[[Server]]
+Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events"
+Response.Body = '''
+{"events": [
+  {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/tests/test_orders.py::test_order_total"}, "details": {"test_case_progress": {"status": "PASSED"}}},
+  {"id": "e2", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:02.000Z", "details": {"test_run_summary": {"total": 1, "passed": 1, "failed": 0, "skipped": 0, "errored": 0}}}
+]}
+'''

acceptance/pipelines/test/live/tests/test_orders.py

@@ -0,0 +1,6 @@
+# order_helpers is not on the import path; only its installation as a test dependency provides it.
+from order_helpers import order_total
+
+
+def test_order_total():
+    assert order_total([1, 2, 3]) == 6

acceptance/pipelines/test/live/transformations/orders.py

@@ -0,0 +1,6 @@
+from pyspark import pipelines as dp
+
+
+@dp.materialized_view
+def orders():
+    return spark.range(3)

acceptance/pipelines/test/no-test-sources/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-pipeline-test-no-test-sources
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-test-no-test-sources
+      libraries:
+        - file:
+            path: pipeline_file.py

acceptance/pipelines/test/no-test-sources/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/pipelines/test/no-test-sources/output.txt

@@ -0,0 +1,17 @@
+
+>>> [CLI] pipelines deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-no-test-sources/default/files...
+Created pipelines.my_pipeline
+Files: 3 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID]
+
+=== The bundle now sets testing, so the service error comes with a hint to deploy
+
+>>> musterr [CLI] pipelines test --no-wait
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Error: [PIPELINE_TEST_NO_TEST_SOURCES] The pipeline has no test sources. Set `testing.sources` to the test files or folders to run.
+The bundle sets testing for my_pipeline, but the deployed pipeline has none. Run 'databricks pipelines deploy' first

acceptance/pipelines/test/no-test-sources/pipeline_file.py

@@ -0,0 +1,6 @@
+from pyspark import pipelines as dp
+
+
+@dp.table
+def orders():
+    return spark.range(10)

acceptance/pipelines/test/no-test-sources/script

@@ -0,0 +1,11 @@
+# The pipeline is deployed before the bundle sets testing.
+trace $CLI pipelines deploy
+rm "$OUT_REQUESTS"
+update_file.py databricks.yml "            path: pipeline_file.py" "            path: pipeline_file.py
+      testing:
+        sources:
+          - glob:
+              include: ./tests/**"
+
+title "The bundle now sets testing, so the service error comes with a hint to deploy"
+trace musterr $CLI pipelines test --no-wait

acceptance/pipelines/test/no-test-sources/test.toml

@@ -0,0 +1,10 @@
+# The deployed pipeline has no test sources.
+[[Server]]
+Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates"
+Response.StatusCode = 400
+Response.Body = '''
+{
+    "error_code": "INVALID_PARAMETER_VALUE",
+    "message": "[PIPELINE_TEST_NO_TEST_SOURCES] The pipeline has no test sources. Set `testing.sources` to the test files or folders to run."
+}
+'''

acceptance/pipelines/test/no-test-sources/tests/test_orders.py

@@ -0,0 +1,6 @@
+def test_valid_order():
+    assert True
+
+
+def test_backfill():
+    assert True