acc: record main's behaviour for reference, override and required-field edge cases (#7130)

## Changes

Add four acceptance tests that record how main handles edge cases of
references, overrides and required fields:
- `override/cluster_id_clears_new_cluster_reference`
- `override/preset_clears_forced_schedule_reference`
- `resource_deps/whole_map_reference_hides_entries`
- `validate/required_struct_reference`

## Why

Records today's behaviour before it changes in #7098, so that PR's diff
shows what moves.

## Tests

Goldens generated on main.

This pull request and its description were written by Isaac.

---------

Co-authored-by: Isaac <no-reply@databricks.com>

Author: denik

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml

@@ -0,0 +1,34 @@
+bundle:
+  name: test-bundle
+  cluster_id: existing-cluster
+
+variables:
+  cluster:
+    type: complex
+    default:
+      num_workers: 1
+
+resources:
+  clusters:
+    source:
+      num_workers: 2
+  jobs:
+    job:
+      name: job
+      tasks:
+        # bundle.cluster_id replaces new_cluster. The reference below it must go too.
+        - task_key: scalar_ref
+          new_cluster:
+            num_workers: ${resources.clusters.source.num_workers}
+        # A reference to the whole new_cluster must not bring it back.
+        - task_key: variable_ref
+          new_cluster: ${var.cluster}
+        # A reference to the new_cluster of another job is resolved after the override.
+        - task_key: resource_ref
+          new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster}
+    other:
+      name: other
+      job_clusters:
+        - job_cluster_key: shared
+          new_cluster:
+            num_workers: 3

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle validate -o json
+[
+  {
+    "existing_cluster_id": "existing-cluster",
+    "task_key": "resource_ref"
+  },
+  {
+    "existing_cluster_id": "existing-cluster",
+    "task_key": "scalar_ref"
+  },
+  {
+    "existing_cluster_id": "existing-cluster",
+    "task_key": "variable_ref"
+  }
+]
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created clusters.source
+Created jobs.job
+Created jobs.other
+Files: 1 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script

@@ -0,0 +1,3 @@
+# bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference.
+trace $CLI bundle validate -o json | jq .resources.jobs.job.tasks
+trace $CLI bundle deploy

acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: test-bundle
+
+resources:
+  quality_monitors:
+    src:
+      table_name: main.default.src
+      assets_dir: /Workspace/monitors/src
+      output_schema_name: main.default
+      schedule:
+        quartz_cron_expression: "0 0 12 * * ?"
+        timezone_id: UTC
+    dst:
+      table_name: main.default.dst
+      assets_dir: /Workspace/monitors/dst
+      output_schema_name: main.default
+
+targets:
+  dev:
+    presets:
+      trigger_pause_status: PAUSED
+    resources:
+      quality_monitors:
+        dst:
+          schedule: ${resources.quality_monitors.src.schedule}

acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files...
+Created quality_monitors.dst
+Created quality_monitors.src
+Files: 1 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //tables/main.default.dst/monitor
+{
+  "assets_dir": "/Workspace/monitors/dst",
+  "output_schema_name": "main.default"
+}

acceptance/bundle/override/preset_clears_forced_schedule_reference/script

@@ -0,0 +1,4 @@
+# trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst
+# with a reference to the schedule of src, which is removed too: the request carries no schedule.
+trace $CLI bundle deploy
+trace print_requests.py //tables/main.default.dst/monitor | jq .body

acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml

@@ -0,0 +1 @@
+RecordRequests = true

acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle
+
+presets:
+  name_prefix: "pre-"
+
+resources:
+  jobs:
+    base:
+      name: base
+    viaref:
+      name: ${resources.jobs.base.name}