diff --git a/.nextchanges/bundles/for-each-task-yaml-inputs.md b/.nextchanges/bundles/for-each-task-yaml-inputs.md new file mode 100644 index 00000000000..5ee5220d05c --- /dev/null +++ b/.nextchanges/bundles/for-each-task-yaml-inputs.md @@ -0,0 +1 @@ +* Support literal YAML arrays directly in job for-each task inputs. Use the JSON string form when inputs need variable interpolation. ([#7304](https://github.com/databricks/cli/pull/7304)) diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml new file mode 100644 index 00000000000..25c57c9317c --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: for_each_inputs_array_reference + +variables: + item: + default: first + +resources: + jobs: + fan: + tasks: + - task_key: array_reference + for_each_task: + inputs: + - name: ${var.item} + task: + task_key: inner + notebook_task: + notebook_path: /notebook diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt new file mode 100644 index 00000000000..c66a6f0fa67 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle validate +Error: failed to rewrite for_each_task.inputs in [TEST_TMP_DIR]/databricks.yml: resources.jobs.fan.tasks[0].for_each_task.inputs at [TEST_TMP_DIR]/databricks.yml:15:15 contains variable reference ${var.item} in a YAML array; use a JSON string for interpolated inputs + + +Found 1 error diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script new file mode 100644 index 00000000000..277355fe918 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script @@ -0,0 +1 @@ +musterr trace $CLI bundle validate diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml new file mode 100644 index 00000000000..e9cede5e28a --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml @@ -0,0 +1,4 @@ +# Errors for invalid YAML print the full path of the file; normalize Windows separators. +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl new file mode 100644 index 00000000000..bb967028c4c --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl @@ -0,0 +1,67 @@ +bundle: + name: for_each_inputs + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME/${bundle.target} + +variables: + inputs: + default: '[ "from_variable", "unchanged_spacing" ]' + item: + default: from_reference + +resources: + jobs: + fan: + name: for-each-inputs-$UNIQUE_NAME-${bundle.target} + tasks: + - task_key: array + for_each_task: + inputs: [root_first, root_second] + task: + task_key: array_inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + - task_key: objects + for_each_task: + inputs: + - name: 'quoted "name" with \backslash' + count: 3 + enabled: true + - name: second + count: 0 + enabled: false + task: + task_key: objects_inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + - task_key: string_variable + for_each_task: + inputs: ${var.inputs} + task: + task_key: string_inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + - task_key: string_interpolation + for_each_task: + inputs: '["${var.item}_x", "y"]' + task: + task_key: interpolation_inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + +targets: + default: + default: true + override: + resources: + jobs: + fan: + tasks: + - task_key: array + for_each_task: + inputs: [target_only] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py b/acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py new file mode 100644 index 00000000000..5f2f3c08ddd --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("for-each inputs") diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt new file mode 100644 index 00000000000..8a4c0b6f1ba --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt @@ -0,0 +1,214 @@ + +>>> [CLI] bundle validate -o json +[ + { + "task_key": "array", + "inputs": "[\"root_first\",\"root_second\"]" + }, + { + "task_key": "objects", + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]" + }, + { + "task_key": "string_interpolation", + "inputs": "[\"from_reference_x\", \"y\"]" + }, + { + "task_key": "string_variable", + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files... +Created jobs.fan +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //jobs --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "for-each-inputs-[UNIQUE_NAME]-default", + "queue": { + "enabled": true + }, + "tasks": [ + { + "for_each_task": { + "inputs": "[\"root_first\",\"root_second\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "array_inner" + } + }, + "task_key": "array" + }, + { + "for_each_task": { + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "objects_inner" + } + }, + "task_key": "objects" + }, + { + "for_each_task": { + "inputs": "[\"from_reference_x\", \"y\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "interpolation_inner" + } + }, + "task_key": "string_interpolation" + }, + { + "for_each_task": { + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "string_inner" + } + }, + "task_key": "string_variable" + } + ] + } +} + +>>> [CLI] bundle validate -o json -t override +[ + { + "task_key": "array", + "inputs": "[\"target_only\"]" + }, + { + "task_key": "objects", + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]" + }, + { + "task_key": "string_interpolation", + "inputs": "[\"from_reference_x\", \"y\"]" + }, + { + "task_key": "string_variable", + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]" + } +] + +>>> [CLI] bundle deploy -t override +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files... +Created jobs.fan +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //jobs --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "for-each-inputs-[UNIQUE_NAME]-override", + "queue": { + "enabled": true + }, + "tasks": [ + { + "for_each_task": { + "inputs": "[\"target_only\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "array_inner" + } + }, + "task_key": "array" + }, + { + "for_each_task": { + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "objects_inner" + } + }, + "task_key": "objects" + }, + { + "for_each_task": { + "inputs": "[\"from_reference_x\", \"y\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "interpolation_inner" + } + }, + "task_key": "string_interpolation" + }, + { + "for_each_task": { + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "string_inner" + } + }, + "task_key": "string_variable" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve -t override +The following resources will be deleted: + delete resources.jobs.fan + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override + +Destroy: 1 deleted + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.fan + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/script b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script new file mode 100644 index 00000000000..ad012477038 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle validate -o json | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' + +trace $CLI bundle deploy +trace print_requests.py //jobs --nostamp + +trace $CLI bundle validate -o json -t override | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' + +trace $CLI bundle deploy -t override +trace print_requests.py //jobs --nostamp + +trace $CLI bundle destroy --auto-approve -t override +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml new file mode 100644 index 00000000000..cf24223e69e --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml @@ -0,0 +1,56 @@ +bundle: + name: for_each_inputs_variable_containers + +variables: + each: + type: complex + default: + inputs: '["from_each"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + task: + type: complex + default: + task_key: fan + for_each_task: + inputs: '["from_task"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + tasks: + type: complex + default: + - task_key: fan + for_each_task: + inputs: '["from_tasks"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + job: + type: complex + default: + tasks: + - task_key: fan + for_each_task: + inputs: '["from_job"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + +resources: + jobs: + each: + tasks: + - task_key: fan + for_each_task: ${var.each} + task: + tasks: + - ${var.task} + tasks: + tasks: ${var.tasks} + job: ${var.job} diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt new file mode 100644 index 00000000000..7574845f2aa --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle validate -o json +[ + { + "job": "each", + "inputs": "[\"from_each\"]" + }, + { + "job": "job", + "inputs": "[\"from_job\"]" + }, + { + "job": "task", + "inputs": "[\"from_task\"]" + }, + { + "job": "tasks", + "inputs": "[\"from_tasks\"]" + } +] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script new file mode 100644 index 00000000000..41a6b09cf0c --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '[.resources.jobs | to_entries[] | {job: .key, inputs: .value.tasks[0].for_each_task.inputs}] | sort_by(.job)' diff --git a/bundle/config/root.go b/bundle/config/root.go index a64b742630e..a2577cdec65 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -3,6 +3,7 @@ package config import ( "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/dynloc" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/log" @@ -130,6 +132,10 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { if err != nil { return nil, diag.Errorf("failed to rewrite %s: %v", path, err) } + v, err = rewriteForEachTaskInputs(v) + if err != nil { + return nil, diag.FromErr(fmt.Errorf("failed to rewrite for_each_task.inputs in %s: %w", path, err)) + } // Normalize dynamic configuration tree according to configuration type. v, diags := convert.Normalize(r, v) @@ -552,6 +558,53 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { })) } +// rewriteForEachTaskInputs encodes literal arrays before SDK string normalization. +// A resource-level any shadow cannot cover this nested SDK field, so this runs +// before typed normalization, unlike resource mutators after variable resolution. +// Other kinds use normal validation; array references fail here. +func rewriteForEachTaskInputs(v dyn.Value) (dyn.Value, error) { + inputs := dyn.NewPattern( + dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), + dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("inputs"), + ) + paths := &dyn.TrieNode{} + for _, prefix := range []dyn.Pattern{ + nil, + dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("environments"), dyn.AnyKey()), + } { + if err := paths.Insert(prefix.Append(inputs...)); err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to match for_each_task.inputs: %w", err) + } + } + return dyn.Walk(v, func(p dyn.Path, input dyn.Value) (dyn.Value, error) { + if input.Kind() != dyn.KindSequence { + return input, nil + } + if _, ok := paths.SearchPath(p); !ok { + return input, nil + } + + array, err := dyn.Walk(input, func(_ dyn.Path, value dyn.Value) (dyn.Value, error) { + if value.Kind() == dyn.KindTime { + return dyn.NewValue(value.MustTime().String(), value.Locations()), nil + } + return value, nil + }) + if err != nil { + return dyn.InvalidValue, err + } + data, err := json.Marshal(array.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("%s at %s: %w", p, input.Location(), err) + } + if ref, ok := dynvar.NewRef(dyn.V(string(data))); ok { + return dyn.InvalidValue, fmt.Errorf("%s at %s contains variable reference ${%s} in a YAML array; use a JSON string for interpolated inputs", p, input.Location(), ref.References()[0]) + } + return dyn.NewValue(string(data), input.Locations()), nil + }) +} + // validateVariableOverrides checks that all variables specified // in the target override are also defined in the root. func validateVariableOverrides(root, target dyn.Value) (err error) { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index f735c44405e..8ce561c8117 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -31,6 +31,192 @@ func TestRootLoad(t *testing.T) { assert.Equal(t, "basic", root.Bundle.Name) } +func TestLoadForEachTaskInputs(t *testing.T) { + for _, tc := range []struct { + name string + inputs string + want string + }{ + {name: "YAML array", inputs: "[input_1, input_2]", want: `["input_1","input_2"]`}, + {name: "YAML object array", inputs: "[{name: input_1, enabled: true}]", want: `[{"enabled":true,"name":"input_1"}]`}, + {name: "empty YAML array", inputs: "[]", want: `[]`}, + {name: "YAML dates", inputs: "[2024-01-01, 2024-01-02]", want: `["2024-01-01","2024-01-02"]`}, + {name: "nested YAML timestamp", inputs: "[{when: 2024-01-01T01:02:03.123+02:00}]", want: `[{"when":"2024-01-01T01:02:03.123+02:00"}]`}, + {name: "quoted YAML elements", inputs: `["a\"b", "line\nbreak"]`, want: `["a\"b","line\nbreak"]`}, + {name: "escaped reference in YAML array", inputs: `["$${var.item}"]`, want: `["$${var.item}"]`}, + {name: "JSON string", inputs: `'[ "input_1", "input_2" ]'`, want: `[ "input_1", "input_2" ]`}, + {name: "variable reference string", inputs: `${var.items}`, want: `${var.items}`}, + } { + t.Run(tc.name, func(t *testing.T) { + config := "bundle:\n name: foreach\nresources:\n jobs:\n fan:\n tasks:\n - task_key: fan\n for_each_task:\n inputs: " + tc.inputs + "\n task:\n task_key: inner\n" + root, diags := LoadFromBytes("databricks.yml", []byte(config)) + require.NoError(t, diags.Error()) + inputs := root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs + require.Equal(t, tc.want, inputs) + + body, err := json.Marshal(root.Resources.Jobs["fan"].Tasks[0].ForEachTask) + require.NoError(t, err) + var apiTask struct { + Inputs string `json:"inputs"` + } + require.NoError(t, json.Unmarshal(body, &apiTask)) + assert.Equal(t, tc.want, apiTask.Inputs) + }) + } +} + +func TestLoadForEachTaskInputsRejectsArrayReferences(t *testing.T) { + for _, tc := range []struct { + name string + inputs string + prefix string + target bool + }{ + {name: "root element", inputs: `["${var.item}_x", y]`, prefix: "resources.jobs.fan"}, + {name: "nested object value", inputs: `[{name: "${var.item}"}]`, prefix: "resources.jobs.fan"}, + {name: "target override", inputs: `["${var.item}"]`, prefix: "targets.dev.resources.jobs.fan", target: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var config string + if tc.target { + config = "bundle:\n name: foreach\nvariables:\n item:\n default: input_1\ntargets:\n dev:\n resources:\n jobs:\n fan:\n tasks:\n - task_key: fan\n for_each_task:\n inputs: " + tc.inputs + "\n task:\n task_key: inner\n" + } else { + config = "bundle:\n name: foreach\nvariables:\n item:\n default: input_1\nresources:\n jobs:\n fan:\n tasks:\n - task_key: fan\n for_each_task:\n inputs: " + tc.inputs + "\n task:\n task_key: inner\n" + } + root, diags := LoadFromBytes("databricks.yml", []byte(config)) + require.Nil(t, root) + require.ErrorContains(t, diags.Error(), "variable reference ${var.item} in a YAML array") + assert.ErrorContains(t, diags.Error(), tc.prefix+".tasks[0].for_each_task.inputs") + assert.ErrorContains(t, diags.Error(), "databricks.yml:") + }) + } +} + +func TestLoadForEachTaskInputsInTarget(t *testing.T) { + config := `bundle: + name: foreach +resources: + jobs: + fan: + tasks: + - task_key: fan + for_each_task: + inputs: [root_1] + task: + task_key: inner +targets: + dev: + resources: + jobs: + fan: + tasks: + - task_key: fan + for_each_task: + inputs: [input_1, input_2] + task: + task_key: inner +` + root, diags := LoadFromBytes("databricks.yml", []byte(config)) + require.NoError(t, diags.Error()) + require.Equal(t, `["root_1"]`, root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) + require.Equal(t, `["input_1","input_2"]`, root.Targets["dev"].Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) + require.NoError(t, root.MergeTargetOverrides("dev")) + require.Len(t, root.Resources.Jobs["fan"].Tasks, 2) + assert.Equal(t, `["root_1"]`, root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) + assert.Equal(t, `["input_1","input_2"]`, root.Resources.Jobs["fan"].Tasks[1].ForEachTask.Inputs) +} + +func TestLoadForEachTaskInputsPreservesVariableContainers(t *testing.T) { + for _, tc := range []struct { + name string + config string + }{ + { + name: "for-each task reference", + config: `bundle: + name: foreach +variables: + each: + type: complex + default: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: + tasks: + - task_key: fan + for_each_task: ${var.each} +`, + }, + { + name: "task reference", + config: `bundle: + name: foreach +variables: + task: + type: complex + default: + task_key: fan + for_each_task: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: + tasks: + - ${var.task} +`, + }, + { + name: "task list reference", + config: `bundle: + name: foreach +variables: + tasks: + type: complex + default: + - task_key: fan + for_each_task: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: + tasks: ${var.tasks} +`, + }, + { + name: "job reference", + config: `bundle: + name: foreach +variables: + job: + type: complex + default: + tasks: + - task_key: fan + for_each_task: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: ${var.job} +`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root, diags := LoadFromBytes("databricks.yml", []byte(tc.config)) + require.NoError(t, diags.Error()) + require.NotNil(t, root) + }) + } +} + func TestInitializeVariables(t *testing.T) { fooDefault := "abc" root := &Root{ diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 01a743933b2..13b50ea1a09 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -120,6 +120,21 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { return s } +func allowForEachTaskInputArrays(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + if typ != reflect.TypeFor[jobs.ForEachTask]() { + return s + } + + inputs := s.Properties["inputs"] + stringRef := inputs.Reference + inputs.Reference = nil + inputs.OneOf = []jsonschema.Schema{ + {Reference: stringRef}, + {Type: jsonschema.ArrayType}, + } + return s +} + func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { switch typ { case reflect.TypeFor[resources.Pipeline](): @@ -287,6 +302,7 @@ func generateSchema(workdir, outputFile, cliJSONFile string, docsMode bool) { removeDeploymentFields, makeVolumeTypeOptional, a.addAnnotations, + allowForEachTaskInputArrays, removeOutputOnlyFields, } if !docsMode { diff --git a/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml b/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml new file mode 100644 index 00000000000..025c0b8039d --- /dev/null +++ b/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml @@ -0,0 +1,64 @@ +bundle: + name: for_each_task_inputs + +variables: + inputs: + default: '["from_variable"]' + +resources: + jobs: + fan: + name: for_each_task_inputs + tasks: + - task_key: array + for_each_task: + inputs: [first, second] + task: + task_key: array_inner + notebook_task: + notebook_path: /notebook + - task_key: objects + for_each_task: + inputs: + - name: first + count: 3 + enabled: true + task: + task_key: objects_inner + notebook_task: + notebook_path: /notebook + - task_key: empty + for_each_task: + inputs: [] + task: + task_key: empty_inner + notebook_task: + notebook_path: /notebook + - task_key: string + for_each_task: + inputs: '["first", "second"]' + task: + task_key: string_inner + notebook_task: + notebook_path: /notebook + - task_key: variable + for_each_task: + inputs: ${var.inputs} + task: + task_key: variable_inner + notebook_task: + notebook_path: /notebook + +targets: + override: + resources: + jobs: + fan: + tasks: + - task_key: array + for_each_task: + inputs: [overridden] + task: + task_key: array_inner + notebook_task: + notebook_path: /notebook diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 959273d2586..f2b5c48ba73 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -10032,8 +10032,15 @@ }, "inputs": { "description": "Array for task to iterate on. This can be a JSON string or a reference to\nan array parameter.", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "GA" + "x-databricks-launch-stage": "GA", + "oneOf": [ + { + "$ref": "#/$defs/string" + }, + { + "type": "array" + } + ] }, "task": { "description": "Configuration for the task that will be run for each element in the array", diff --git a/python/codegen/codegen/jsonschema.py b/python/codegen/codegen/jsonschema.py index 9ad7191a55c..72c8b40eceb 100644 --- a/python/codegen/codegen/jsonschema.py +++ b/python/codegen/codegen/jsonschema.py @@ -91,6 +91,20 @@ def _unwrap_variable(schema: dict): return None +def _property_ref(value: dict) -> Optional[str]: + if ref := value.get("$ref"): + return ref + + # CLI accepts literal YAML arrays here, while PyDABs still emits the SDK string. + if value.get("oneOf") == [ + {"$ref": "#/$defs/string"}, + {"type": "array"}, + ]: + return "#/$defs/string" + + return None + + def _parse_schema(schema: dict) -> Schema: schema = _unwrap_variable(schema) or schema properties = {} @@ -106,10 +120,11 @@ def _parse_bool(value) -> Optional[bool]: assert v.get("properties") is None assert v.get("items") is None - assert v.get("$ref") + ref = _property_ref(v) + assert ref prop = Property( - ref=v["$ref"], + ref=ref, description=v.get("description"), deprecated=_parse_bool(v.get("deprecated")), stage=v.get("x-databricks-launch-stage"), @@ -179,9 +194,9 @@ def _refs_in_raw_schema(schema: dict) -> list[str]: schema = _unwrap_variable(schema) or schema return [ - prop["$ref"].split("/")[-1] + ref.split("/")[-1] for prop in schema.get("properties", {}).values() - if prop.get("$ref") + if (ref := _property_ref(prop)) ] diff --git a/python/codegen/codegen_tests/test_jsonschema.py b/python/codegen/codegen_tests/test_jsonschema.py index 307bcc1dec9..d19c7f9e508 100644 --- a/python/codegen/codegen_tests/test_jsonschema.py +++ b/python/codegen/codegen_tests/test_jsonschema.py @@ -3,6 +3,7 @@ Schema, SchemaType, _parse_schema, + _refs_in_raw_schema, _unwrap_variable, get_schemas, ) @@ -77,3 +78,26 @@ def test_parse_schema(): ) assert _parse_schema(spec) == expected + + +def test_parse_schema_for_each_inputs_array_shorthand(): + spec = { + "type": "object", + "properties": { + "inputs": { + "description": "For-each task inputs", + "x-databricks-launch-stage": "GA", + "oneOf": [ + {"$ref": "#/$defs/string"}, + {"type": "array"}, + ], + } + }, + } + + assert _parse_schema(spec).properties["inputs"] == Property( + ref="#/$defs/string", + description="For-each task inputs", + stage="GA", + ) + assert _refs_in_raw_schema(spec) == ["string"]