Add bundle bitmap for per-field telemetry

.agents/rules/telemetry.md

@@ -34,7 +34,7 @@ The deploy payload is declared twice: the proto in the universe repo (`proto/log
 
 Other things worth knowing:
 
-- A one-off boolean needs no proto change: add a key constant to `bundle/metrics/metrics.go` (snake_case, matching the constant name) and call `b.Metrics.SetBoolValue(...)`. It lands in `experimental.bool_values`.
+- A one-off boolean needs no proto change: add a field to `bundle.Telemetry` (`bundle/telemetry.go`) with a snake_case `metric:"..."` tag and set it. Pick the type by what is measured: `bool` for a flag reported only when true, `*bool` for a measured value (nil when not measured), `[]bool` for one value per item. It lands in `experimental.bool_values` and in the bundle bitmap; run `./task generate-bitmap-schema` to append it to the bitmap schema.
 - Metrics that may be removed later belong in `BundleDeployExperimental`, which carries no compatibility guarantees.
 - Positional encodings such as the `upload_file_sizes` histogram are frozen once the metric has adoption: changing a bucket bound silently changes the meaning of every previously reported entry.
 - Per-resource state sizes come from the direct engine's state in `resources.json`.

Taskfile.yml

@@ -802,6 +802,7 @@ tasks:
       # Refreshes acceptance/bundle/refschema/output/*.txt, which feeds
       # generate-direct-apitypes and generate-direct-resources below.
       - task: generate-refschema
+      - task: generate-bitmap-schema
       - task: generate-schema
       - task: generate-validation
       - task: generate-direct
@@ -820,6 +821,7 @@ tasks:
     cmds:
       - task: generate-cligen
       - task: generate-refschema
+      - task: generate-bitmap-schema
       - task: generate-schema
       - task: generate-validation
       - task: generate-direct
@@ -989,6 +991,32 @@ tasks:
     cmds:
       - go test ./acceptance -run TestAccept/bundle/refschema -update
 
+  # Appends any newly added bundle config fields to bundle/bitmap/schema.txt, the
+  # embedded bitmap schema (see cmd/bundle/bitmap). The schema is append-only, so
+  # update-schema reads the currently embedded schema and only appends. Output is
+  # written to a temp file first so the redirect does not truncate schema.txt
+  # before `go run` embeds it at compile time. Also refreshes the schema embedded
+  # in the databricks-bundles decoder (python/databricks/bundles/_bitmap) and the
+  # listing in acceptance/bundle/bitmap/full, which tracks every schema line.
+  generate-bitmap-schema:
+    desc: Regenerate bundle/bitmap/schema.txt and its acceptance listing
+    sources:
+      - bundle/**/*.go
+      - exclude: bundle/**/*_test.go
+      - acceptance/bundle/bitmap/full/script
+      - go.mod
+      - go.sum
+      - "{{.EMBED_SOURCES}}"
+    generates:
+      - bundle/bitmap/schema.txt
+      - acceptance/bundle/bitmap/full/out.schema.txt
+      - python/databricks/bundles/_bitmap/_schema.py
+    cmds:
+      - go run . bundle bitmap update-schema > bundle/bitmap/schema.txt.tmp
+      - mv bundle/bitmap/schema.txt.tmp bundle/bitmap/schema.txt
+      - go run ./bundle/bitmap/genpython python/databricks/bundles/_bitmap/_schema.py
+      - go test ./acceptance -run TestAccept/bundle/bitmap/full -update
+
   # Upstream field documentation comes from the checked-in .codegen/cli.json;
   # bundle/internal/schema/annotations.yml carries the CLI-owned docs and
   # overrides and is rewritten in place (synced with the config structure).

acceptance/bundle/bitmap/full/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = []

acceptance/bundle/bitmap/full/output.txt

@@ -0,0 +1,2 @@
+
+>>> [CLI] bundle bitmap schema

acceptance/bundle/bitmap/full/script

@@ -0,0 +1,4 @@
+# Records every schema line, so a PR that adds a config field shows where it
+# landed in the append-only schema. Refreshed by 'task generate-bitmap-schema';
+# ../presence pins the semantics of the bits.
+trace $CLI bundle bitmap schema > out.schema.txt

acceptance/bundle/bitmap/full/test.toml

@@ -0,0 +1,2 @@
+# The schema does not depend on the bundle or the DMS variant.
+EnvMatrix.DMS = []

acceptance/bundle/bitmap/presence/databricks.yml

@@ -0,0 +1,79 @@
+bundle:
+  name: test-bundle
+
+variables:
+  scalar_var:
+    description: a variable
+    default: abc
+  complex_var:
+    type: complex
+    default:
+      key: value
+  flag:
+    default: true
+
+# Known gap: an explicit false on an omitempty bool of a CLI-defined struct (11
+# such fields, all without ForceSendFields) is dropped when the loader syncs typed
+# config back to dyn, so only "experimental" itself is recorded here. SDK bools
+# keep it (see disable_auto_optimization below).
+experimental:
+  use_legacy_run_as: false
+
+presets:
+  name_prefix: dev_
+  pipelines_development: false
+
+run_as:
+  service_principal_name: sp-1234
+
+artifacts:
+  # One artifact with a build command and one with files only, so each
+  # per-artifact flag records both outcomes.
+  with_build:
+    type: whl
+    build: "true"
+  with_files:
+    type: whl
+    files:
+      - source: wheels/*.whl
+
+resources:
+  jobs:
+    my_job:
+      name: My Job
+      description: "escaped $${var.scalar_var}"
+      timeout_seconds: 0
+      email_notifications: {}
+      parameters: []
+      tags:
+        team: data
+        "a.b": c
+      # Unresolved when the bitmap snapshots the config: present, with no value bit.
+      notification_settings:
+        no_alert_for_skipped_runs: ${var.flag}
+      # true here and false in other_job: both value bits are set.
+      queue:
+        enabled: true
+      tasks:
+        - task_key: a
+          # SDK bool with omitempty: an explicit false is kept via ForceSendFields.
+          disable_auto_optimization: false
+          notebook_task:
+            notebook_path: ./notebook.py
+        - task_key: b
+          depends_on:
+            - task_key: a
+          notebook_task:
+            notebook_path: ./notebook.py
+    other_job:
+      name: Other Job
+      queue:
+        enabled: false
+
+targets:
+  dev:
+    default: true
+    resources:
+      jobs:
+        my_job:
+          max_concurrent_runs: 2

acceptance/bundle/bitmap/presence/invalid/databricks.yml

@@ -0,0 +1,5 @@
+bundle:
+  name: test-bundle
+
+artifacts:
+  broken:

acceptance/bundle/bitmap/presence/notebook.py

@@ -0,0 +1 @@
+# Databricks notebook source

acceptance/bundle/bitmap/presence/out.bitmap.txt

@@ -0,0 +1,57 @@
+1 variables
+1 variables.*
+1 variables.*.type
+1 variables.*.default
+1 variables.*.description
+1 bundle
+1 bundle.name
+1 bundle.target
+1 bundle.environment
+1 artifacts
+1 artifacts.*
+1 artifacts.*.type
+1 artifacts.*.files
+1 artifacts.*.files[*]
+1 artifacts.*.files[*].source
+1 artifacts.*.build
+1 resources
+1 resources.jobs
+1 resources.jobs.*
+1 resources.jobs.*.description
+1 resources.jobs.*.email_notifications
+1 resources.jobs.*.max_concurrent_runs
+1 resources.jobs.*.name
+1 resources.jobs.*.notification_settings
+1 resources.jobs.*.notification_settings.no_alert_for_skipped_runs
+1 resources.jobs.*.parameters
+1 resources.jobs.*.queue
+T resources.jobs.*.queue.enabled
+F resources.jobs.*.queue.enabled
+1 resources.jobs.*.tags
+1 resources.jobs.*.tags.*
+1 resources.jobs.*.tasks
+1 resources.jobs.*.tasks[*]
+1 resources.jobs.*.tasks[*].depends_on
+1 resources.jobs.*.tasks[*].depends_on[*]
+1 resources.jobs.*.tasks[*].depends_on[*].task_key
+F resources.jobs.*.tasks[*].disable_auto_optimization
+1 resources.jobs.*.tasks[*].notebook_task
+1 resources.jobs.*.tasks[*].notebook_task.notebook_path
+1 resources.jobs.*.tasks[*].task_key
+1 resources.jobs.*.timeout_seconds
+1 run_as
+1 run_as.service_principal_name
+1 presets
+1 presets.name_prefix
+F presets.pipelines_development
+1 experimental
+1 telemetry.ArtifactBuildCommandIsSet
+T telemetry.ArtifactBuildCommandIsSet[*]
+F telemetry.ArtifactBuildCommandIsSet[*]
+1 telemetry.ArtifactFilesIsSet
+T telemetry.ArtifactFilesIsSet[*]
+F telemetry.ArtifactFilesIsSet[*]
+T telemetry.PresetsNamePrefixIsSet
+T telemetry.ConfigHasDoubleDollarBrace
+F telemetry.UseLegacyRunAs
+T telemetry.RunAsSet

acceptance/bundle/bitmap/presence/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/bitmap/presence/output.txt

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle bitmap bitmap-text
+
+>>> [CLI] bundle bitmap bitmap
+
+>>> [CLI] bundle bitmap decode
+
+>>> uv run --isolated --no-project --reinstall-package databricks-bundles --with [DATABRICKS_BUNDLES_WHEEL] -q python -m databricks.bundles._bitmap
+
+>>> [CLI] bundle bitmap update-schema --validate
+
+=== decode rejects input that is not an encoded bitmap
+
+>>> musterr [CLI] bundle bitmap decode
+Error: base64: illegal base64 data at input byte 3
+
+=== An invalid bundle fails instead of printing a bitmap
+
+>>> musterr [CLI] bundle bitmap bitmap
+Error: Artifact not properly configured
+  in databricks.yml:5:10
+
+please specify artifact properties
+

acceptance/bundle/bitmap/presence/script

@@ -0,0 +1,23 @@
+# One line per set field ("1", or "T"/"F" for a bool), so a field appended to the
+# schema changes nothing here (../full records the whole schema).
+trace $CLI bundle bitmap bitmap-text > out.bitmap.txt
+
+# The encoded bitmap decodes back to the same fields, with the CLI and with the
+# self-contained decoder in databricks-bundles. --reinstall-package: the wheel's
+# version does not change with the code, and uv would reuse a cached install.
+trace $CLI bundle bitmap bitmap > bitmap.b64
+trace $CLI bundle bitmap decode < bitmap.b64 > decoded.txt
+diff.py out.bitmap.txt decoded.txt
+trace uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap < bitmap.b64 > decoded-python.txt
+diff.py out.bitmap.txt decoded-python.txt
+
+trace $CLI bundle bitmap update-schema --validate
+
+title "decode rejects input that is not an encoded bitmap\n"
+echo not-a-bitmap | trace musterr $CLI bundle bitmap decode
+# The message comes from Python's base64 module and varies by version, so it is logged, not diffed.
+echo not-a-bitmap | trace musterr uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap 2> LOG.decode_python
+
+title "An invalid bundle fails instead of printing a bitmap\n"
+cd invalid
+trace musterr $CLI bundle bitmap bitmap

acceptance/bundle/bitmap/presence/test.toml

@@ -0,0 +1,2 @@
+# Scratch files of the round-trip check; out.bitmap.txt is the golden.
+Ignore = [".databricks", "bitmap.b64", "decoded.txt", "decoded-python.txt"]

acceptance/bundle/invariant/validate_bitmap/out.test.toml

@@ -0,0 +1,69 @@
+Cloud = false
+EnvMatrix.DMS = []
+EnvMatrix.INPUT_CONFIG = [
+  "alert.yml.tmpl",
+  "app.yml.tmpl",
+  "catalog.yml.tmpl",
+  "catalog_optional_fields.yml.tmpl",
+  "cluster.yml.tmpl",
+  "cluster_apply_policy_default_values.yml.tmpl",
+  "cluster_libraries.yml.tmpl",
+  "cluster_policy.yml.tmpl",
+  "dashboard.yml.tmpl",
+  "job_apply_policy_default_values_job_cluster.yml.tmpl",
+  "job_apply_policy_default_values_task_cluster.yml.tmpl",
+  "job_apply_policy_default_values_for_each_task.yml.tmpl",
+  "data_quality_monitor.yml.tmpl",
+  "database_catalog.yml.tmpl",
+  "database_instance.yml.tmpl",
+  "experiment.yml.tmpl",
+  "external_location.yml.tmpl",
+  "feature.yml.tmpl",
+  "genie_space.yml.tmpl",
+  "instance_pool.yml.tmpl",
+  "job.yml.tmpl",
+  "job_pydabs_10_tasks.yml.tmpl",
+  "job_pydabs_1000_tasks.yml.tmpl",
+  "job_cross_resource_ref.yml.tmpl",
+  "job_escaped_refs.yml.tmpl",
+  "job_permission_ref.yml.tmpl",
+  "job_run.yml.tmpl",
+  "job_run_job_ref.yml.tmpl",
+  "job_table_update_trigger.yml.tmpl",
+  "job_with_depends_on.yml.tmpl",
+  "job_with_task.yml.tmpl",
+  "model.yml.tmpl",
+  "model_with_permissions.yml.tmpl",
+  "model_serving_endpoint.yml.tmpl",
+  "model_serving_endpoint_telemetry.yml.tmpl",
+  "pipeline.yml.tmpl",
+  "pipeline_allow_duplicate_names.yml.tmpl",
+  "pipeline_apply_policy_default_values.yml.tmpl",
+  "pipeline_config_dots.yml.tmpl",
+  "postgres_branch.yml.tmpl",
+  "postgres_catalog.yml.tmpl",
+  "postgres_database.yml.tmpl",
+  "postgres_endpoint.yml.tmpl",
+  "postgres_project.yml.tmpl",
+  "postgres_role.yml.tmpl",
+  "postgres_synced_table.yml.tmpl",
+  "registered_model.yml.tmpl",
+  "schema.yml.tmpl",
+  "schema_tf_compat.yml.tmpl",
+  "schema_empty_grants.yml.tmpl",
+  "schema_grant_ref.yml.tmpl",
+  "schema_uppercase_name.yml.tmpl",
+  "secret.yml.tmpl",
+  "secret_scope.yml.tmpl",
+  "secret_scope_default_backend_type.yml.tmpl",
+  "sql_warehouse.yml.tmpl",
+  "synced_database_table.yml.tmpl",
+  "uc_trailing_slash.yml.tmpl",
+  "vector_search_endpoint.yml.tmpl",
+  "vector_search_index.yml.tmpl",
+  "volume.yml.tmpl",
+  "volume_external.yml.tmpl",
+  "volume_path_job_ref.yml.tmpl",
+  "volume_uppercase_name.yml.tmpl",
+  "model_service.yml.tmpl"
+]

acceptance/bundle/invariant/validate_bitmap/output.txt

@@ -0,0 +1 @@
+INPUT_CONFIG_OK

acceptance/bundle/invariant/validate_bitmap/script

@@ -0,0 +1,30 @@
+# Invariant to test: every config validates, and its bitmap survives encoding: both the CLI decoder
+# and the databricks-bundles Python decoder reproduce the bits computed before encoding.
+# Nothing is deployed, so there is nothing to clean up.
+
+invariant_render
+
+# Validation output differs per config: log it, check for panics.
+$CLI bundle validate &> LOG.validate
+cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null
+$CLI bundle validate -o json > LOG.validate_json 2> LOG.validate_json_err
+cat LOG.validate_json_err | contains.py '!panic:' '!internal error' > /dev/null
+
+# Tells the fuzzer the generated config was valid; failures after this count as bugs.
+echo INPUT_CONFIG_OK
+
+$CLI bundle bitmap bitmap-text > LOG.bitmap_text 2> LOG.bitmap_text_err
+cat LOG.bitmap_text_err | contains.py '!panic:' '!internal error' > /dev/null
+# Every config names its bundle; guards against comparing two empty decodes.
+cat LOG.bitmap_text | contains.py '1 bundle.name' > /dev/null
+
+$CLI bundle bitmap bitmap > LOG.bitmap 2> LOG.bitmap_err
+cat LOG.bitmap_err | contains.py '!panic:' '!internal error' > /dev/null
+
+$CLI bundle bitmap decode < LOG.bitmap > LOG.bitmap_decoded
+# --reinstall-package: the wheel's version does not change with the code, and uv would reuse a
+# cached install.
+uv run --isolated --no-project --reinstall-package databricks-bundles --with "$DATABRICKS_BUNDLES_WHEEL" -q python -m databricks.bundles._bitmap < LOG.bitmap > LOG.bitmap_decoded_python
+
+diff.py LOG.bitmap_text LOG.bitmap_decoded
+diff.py LOG.bitmap_text LOG.bitmap_decoded_python

acceptance/bundle/invariant/validate_bitmap/test.toml

@@ -0,0 +1,5 @@
+# Nothing is deployed, so the local test server is enough.
+Cloud = false
+
+# Neither validate nor the bitmap uses the deployment metadata service.
+EnvMatrix.DMS = []

acceptance/bundle/python/telemetry/output.txt

@@ -68,3 +68,9 @@ Resources: 0 created, 0 changed, 0 deleted, 4 unchanged
     "schemas"
   ]
 }
+
+=== Bitmap includes resources defined in Python
+>>> uv run [UV_ARGS] -q [CLI] bundle bitmap bitmap-text
+1 resources.pipelines
+1 resources.pipelines.*
+1 resources.pipelines.*.name