Skip to content

Commit 7767812

Browse files
Default deployment history to enforced file-backed recording
1 parent 1723931 commit 7767812

50 files changed

Lines changed: 1096 additions & 297 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎acceptance/bin/print_state.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -174,23 +174,26 @@ def get_last_version_id(target):
174174

175175

176176
def print_recorded_state(filename, target):
177-
"""Print the state file with its resources filled in from the deployment metadata service.
177+
"""Print DMS resources using the state file's header.
178178
179-
While recording, the file itself carries only the header - the service holds the resources - so
180-
printing it raw would show an empty state and differ from the same test's non-recording run.
179+
This checks history uploads even though resources.json remains authoritative.
180+
Use --no-dms to inspect the file state instead.
181181
"""
182182
data = json.loads(open(filename).read())
183183
# Recording stamps each resource payload with the deployment and version; drop it here so the
184184
# printed state matches a non-recording run without every caller piping through nostamp.
185185
data["state"] = scrub(get_recorded_state(target))
186186

187-
# The file's serial is a copy of the DMS version to ensure outdated resource state is not read.
188-
# Rebuild in header order with the authoritative serial from the deployment.
187+
# Pair DMS resources with the DMS version. The file serial can advance independently.
189188
last_version_id = get_last_version_id(target)
190189
serial = int(last_version_id) if last_version_id else 0
191190

192191
# Recording itself is not what these tests assert, so drop the feature that marks it.
193-
features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"}
192+
features = {
193+
k: v
194+
for k, v in (data.get("features") or {}).items()
195+
if k not in {"deployment_history", "deployment_history_best_effort"}
196+
}
194197

195198
rebuilt = {}
196199
for key in ("state_version", "cli_version", "lineage"):
@@ -213,7 +216,7 @@ def main():
213216
"--no-dms",
214217
action="store_true",
215218
help="Print resources.json as-is without fetching state from DMS. "
216-
"Use in bundle/dms tests to assert the tombstone content of the file.",
219+
"Use in bundle/dms tests to assert the file state and its feature markers.",
217220
)
218221
args = parser.parse_args()
219222

‎acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55
"serial": 0,
66
"plan": {},
77
"features": {
8-
"deployment_history": {}
8+
"deployment_history_best_effort": {}
99
}
1010
}

‎acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
bundle:
22
name: dms-bind-unbind-$UNIQUE_NAME
3-
experimental:
4-
deployment_history: true
5-
63
resources:
74
jobs:
85
foo:

‎acceptance/bundle/dms/declined-deploy/output.txt‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
6767
"lineage": "[UUID]",
6868
"serial": 1,
6969
"features": {
70-
"deployment_history": {}
70+
"deployment_history_best_effort": {}
7171
}
7272
}
7373

@@ -79,18 +79,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
7979
"method": "GET",
8080
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
8181
}
82-
{
83-
"method": "GET",
84-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
85-
}
8682
{
8783
"method": "GET",
8884
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
8985
}
90-
{
91-
"method": "GET",
92-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
93-
}
9486

9587
>>> print_state.py --no-dms
9688
{
@@ -99,9 +91,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
9991
"lineage": "[UUID]",
10092
"serial": 1,
10193
"features": {
102-
"deployment_history": {}
94+
"deployment_history_best_effort": {}
10395
},
104-
"state": {}
96+
"state": {
97+
"resources.schemas.foo": {
98+
"__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]",
99+
"state": {
100+
"catalog_name": "main",
101+
"name": "dms_declined_deploy_schema_[UNIQUE_NAME]"
102+
}
103+
}
104+
}
105105
}
106106

107107
=== A destructive change without --auto-approve is declined: this console cannot prompt
@@ -124,10 +124,6 @@ Files: 2 uploaded, 0 deleted
124124
"method": "GET",
125125
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
126126
}
127-
{
128-
"method": "GET",
129-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
130-
}
131127

132128
=== The recorded resources are unchanged by the decline: the schema is still at version 1
133129
>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
@@ -163,10 +159,6 @@ false
163159
"method": "GET",
164160
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
165161
}
166-
{
167-
"method": "GET",
168-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
169-
}
170162
{
171163
"method": "POST",
172164
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",

‎acceptance/bundle/dms/depends-on/output.txt‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"plan_version": [PLAN_VERSION],
66
"cli_version": "[CLI_VERSION]",
77
"features": {
8-
"deployment_history": {}
8+
"deployment_history_best_effort": {}
99
}
1010
}
1111

@@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
6161
"lineage": "[UUID]",
6262
"serial": 1,
6363
"features": {
64-
"deployment_history": {}
64+
"deployment_history_best_effort": {}
6565
}
6666
}
6767

@@ -91,9 +91,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
9191
"lineage": "[UUID]",
9292
"serial": 1,
9393
"features": {
94-
"deployment_history": {}
94+
"deployment_history_best_effort": {}
9595
},
96-
"state": {}
96+
"state": {
97+
"resources.jobs.child": {
98+
"__id__": "[CHILD_ID]",
99+
"state": {
100+
"deployment": {
101+
"deployment_id": "[DEPLOYMENT_ID]",
102+
"kind": "BUNDLE",
103+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
104+
"version_id": "1"
105+
},
106+
"description": "depends on [PARENT_ID]",
107+
"edit_mode": "UI_LOCKED",
108+
"format": "MULTI_TASK",
109+
"max_concurrent_runs": 1,
110+
"name": "child",
111+
"queue": {
112+
"enabled": true
113+
}
114+
},
115+
"depends_on": [
116+
{
117+
"node": "resources.jobs.parent",
118+
"label": "${resources.jobs.parent.id}"
119+
}
120+
]
121+
},
122+
"resources.jobs.parent": {
123+
"__id__": "[PARENT_ID]",
124+
"state": {
125+
"deployment": {
126+
"deployment_id": "[DEPLOYMENT_ID]",
127+
"kind": "BUNDLE",
128+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
129+
"version_id": "1"
130+
},
131+
"edit_mode": "UI_LOCKED",
132+
"format": "MULTI_TASK",
133+
"max_concurrent_runs": 1,
134+
"name": "parent",
135+
"queue": {
136+
"enabled": true
137+
}
138+
}
139+
}
140+
}
97141
}
98142

99143
>>> [CLI] bundle destroy --auto-approve

‎acceptance/bundle/dms/deployment-metadata-change/output.txt‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
6767
"lineage": "[UUID]",
6868
"serial": 1,
6969
"features": {
70-
"deployment_history": {}
70+
"deployment_history_best_effort": {}
7171
}
7272
}
7373

@@ -114,6 +114,25 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
114114
}
115115
}
116116
}
117+
{
118+
"method": "POST",
119+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
120+
"q": {
121+
"version_id": "2"
122+
},
123+
"body": {
124+
"cli_version": "[CLI_VERSION]",
125+
"previous_version_id": "1",
126+
"version_type": "VERSION_TYPE_DEPLOY"
127+
}
128+
}
129+
{
130+
"method": "POST",
131+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete",
132+
"body": {
133+
"completion_reason": "VERSION_COMPLETE_SUCCESS"
134+
}
135+
}
117136

118137
>>> [CLI] bundle plan -t prod -o json
119138
Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed
@@ -128,7 +147,7 @@ A common practice is to use a username or principal name in this path, i.e. use
128147
"lineage": "[UUID]",
129148
"serial": 1,
130149
"features": {
131-
"deployment_history": {}
150+
"deployment_history_best_effort": {}
132151
}
133152
}
134153

@@ -163,6 +182,25 @@ Files: 0 uploaded, 0 deleted
163182
Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
164183

165184
>>> print_requests.py --dms //api/2.0/bundle
185+
{
186+
"method": "POST",
187+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
188+
"q": {
189+
"version_id": "3"
190+
},
191+
"body": {
192+
"cli_version": "[CLI_VERSION]",
193+
"previous_version_id": "2",
194+
"version_type": "VERSION_TYPE_DEPLOY"
195+
}
196+
}
197+
{
198+
"method": "POST",
199+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete",
200+
"body": {
201+
"completion_reason": "VERSION_COMPLETE_SUCCESS"
202+
}
203+
}
166204

167205
>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
168206
[

‎acceptance/bundle/dms/empty-grants/output.txt‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
138138
"lineage": "[UUID]",
139139
"serial": 2,
140140
"features": {
141-
"deployment_history": {}
141+
"deployment_history_best_effort": {}
142142
}
143143
}
144144

@@ -163,9 +163,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
163163
"lineage": "[UUID]",
164164
"serial": 2,
165165
"features": {
166-
"deployment_history": {}
166+
"deployment_history_best_effort": {}
167167
},
168-
"state": {}
168+
"state": {
169+
"resources.schemas.foo": {
170+
"__id__": "main.dms_empty_grants_[UNIQUE_NAME]",
171+
"state": {
172+
"catalog_name": "main",
173+
"name": "dms_empty_grants_[UNIQUE_NAME]"
174+
}
175+
}
176+
}
169177
}
170178

171179
>>> [CLI] bundle destroy --auto-approve

‎acceptance/bundle/dms/existing-state/output.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
7373
{
7474
"state_version": 3,
7575
"features": {
76-
"deployment_history": {}
76+
"deployment_history_best_effort": {}
7777
}
7878
}

‎acceptance/bundle/dms/existing-state/script‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
1616

1717
title "A leftover local WAL with the wrong lineage is still refused"
1818
mkdir -p .databricks/bundle/default
19-
printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal
19+
printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history_best_effort":{}}}\n' > .databricks/bundle/default/resources.json.wal
2020
musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
2121

2222
title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone"

0 commit comments

Comments
 (0)