Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/for-each-task-yaml-inputs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Support literal YAML arrays directly in job for-each task inputs. Use the JSON string form when inputs need variable interpolation. ([#7304](https://github.com/databricks/cli/pull/7304))
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
bundle:
name: for_each_inputs_array_reference

variables:
item:
default: first

resources:
jobs:
fan:
tasks:
- task_key: array_reference
for_each_task:
inputs:
- name: ${var.item}
task:
task_key: inner
notebook_task:
notebook_path: /notebook
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@

>>> [CLI] bundle validate
Error: failed to rewrite for_each_task.inputs in [TEST_TMP_DIR]/databricks.yml: resources.jobs.fan.tasks[0].for_each_task.inputs at [TEST_TMP_DIR]/databricks.yml:15:15 contains variable reference ${var.item} in a YAML array; use a JSON string for interpolated inputs


Found 1 error
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
musterr trace $CLI bundle validate
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Errors for invalid YAML print the full path of the file; normalize Windows separators.
[[Repls]]
Old = '\\'
New = '/'
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
bundle:
name: for_each_inputs

workspace:
root_path: ~/.bundle/$UNIQUE_NAME/${bundle.target}

variables:
inputs:
default: '[ "from_variable", "unchanged_spacing" ]'
item:
default: from_reference

resources:
jobs:
fan:
name: for-each-inputs-$UNIQUE_NAME-${bundle.target}
tasks:
- task_key: array
for_each_task:
inputs: [root_first, root_second]
task:
task_key: array_inner
notebook_task:
notebook_path: ./notebook.py
source: WORKSPACE
- task_key: objects
for_each_task:
inputs:
- name: 'quoted "name" with \backslash'
count: 3
enabled: true
- name: second
count: 0
enabled: false
task:
task_key: objects_inner
notebook_task:
notebook_path: ./notebook.py
source: WORKSPACE
- task_key: string_variable
for_each_task:
inputs: ${var.inputs}
task:
task_key: string_inner
notebook_task:
notebook_path: ./notebook.py
source: WORKSPACE
- task_key: string_interpolation
for_each_task:
inputs: '["${var.item}_x", "y"]'
task:
task_key: interpolation_inner
notebook_task:
notebook_path: ./notebook.py
source: WORKSPACE

targets:
default:
default: true
override:
resources:
jobs:
fan:
tasks:
- task_key: array
for_each_task:
inputs: [target_only]
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Databricks notebook source
print("for-each inputs")
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = true
EnvMatrix.DMS = ["", "true"]
214 changes: 214 additions & 0 deletions acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@

>>> [CLI] bundle validate -o json
[
{
"task_key": "array",
"inputs": "[\"root_first\",\"root_second\"]"
},
{
"task_key": "objects",
"inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]"
},
{
"task_key": "string_interpolation",
"inputs": "[\"from_reference_x\", \"y\"]"
},
{
"task_key": "string_variable",
"inputs": "[ \"from_variable\", \"unchanged_spacing\" ]"
}
]

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files...
Created jobs.fan
Files: 3 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py //jobs --nostamp
{
"method": "POST",
"path": "/api/2.2/jobs/create",
"body": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "for-each-inputs-[UNIQUE_NAME]-default",
"queue": {
"enabled": true
},
"tasks": [
{
"for_each_task": {
"inputs": "[\"root_first\",\"root_second\"]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook",
"source": "WORKSPACE"
},
"task_key": "array_inner"
}
},
"task_key": "array"
},
{
"for_each_task": {
"inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook",
"source": "WORKSPACE"
},
"task_key": "objects_inner"
}
},
"task_key": "objects"
},
{
"for_each_task": {
"inputs": "[\"from_reference_x\", \"y\"]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook",
"source": "WORKSPACE"
},
"task_key": "interpolation_inner"
}
},
"task_key": "string_interpolation"
},
{
"for_each_task": {
"inputs": "[ \"from_variable\", \"unchanged_spacing\" ]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook",
"source": "WORKSPACE"
},
"task_key": "string_inner"
}
},
"task_key": "string_variable"
}
]
}
}

>>> [CLI] bundle validate -o json -t override
[
{
"task_key": "array",
"inputs": "[\"target_only\"]"
},
{
"task_key": "objects",
"inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]"
},
{
"task_key": "string_interpolation",
"inputs": "[\"from_reference_x\", \"y\"]"
},
{
"task_key": "string_variable",
"inputs": "[ \"from_variable\", \"unchanged_spacing\" ]"
}
]

>>> [CLI] bundle deploy -t override
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files...
Created jobs.fan
Files: 3 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py //jobs --nostamp
{
"method": "POST",
"path": "/api/2.2/jobs/create",
"body": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "for-each-inputs-[UNIQUE_NAME]-override",
"queue": {
"enabled": true
},
"tasks": [
{
"for_each_task": {
"inputs": "[\"target_only\"]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook",
"source": "WORKSPACE"
},
"task_key": "array_inner"
}
},
"task_key": "array"
},
{
"for_each_task": {
"inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook",
"source": "WORKSPACE"
},
"task_key": "objects_inner"
}
},
"task_key": "objects"
},
{
"for_each_task": {
"inputs": "[\"from_reference_x\", \"y\"]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook",
"source": "WORKSPACE"
},
"task_key": "interpolation_inner"
}
},
"task_key": "string_interpolation"
},
{
"for_each_task": {
"inputs": "[ \"from_variable\", \"unchanged_spacing\" ]",
"task": {
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook",
"source": "WORKSPACE"
},
"task_key": "string_inner"
}
},
"task_key": "string_variable"
}
]
}
}

>>> [CLI] bundle destroy --auto-approve -t override
The following resources will be deleted:
delete resources.jobs.fan

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override

Destroy: 1 deleted

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.fan

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default

Destroy: 1 deleted
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
envsubst < databricks.yml.tmpl > databricks.yml

trace $CLI bundle validate -o json | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})'

trace $CLI bundle deploy
trace print_requests.py //jobs --nostamp

trace $CLI bundle validate -o json -t override | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})'

trace $CLI bundle deploy -t override
trace print_requests.py //jobs --nostamp

trace $CLI bundle destroy --auto-approve -t override
trace $CLI bundle destroy --auto-approve
rm "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = true
Loading
Loading