Skip to content

Commit 1941ff7

Browse files
denikIsaac
andcommitted
bundle: close the migration WAL before removing the temp state
A failed Terraform state conversion returned with the temp state's WAL still open, so removing it failed on Windows and left the files until the next run. Add DeploymentState.Discard (close without persisting, remove the file and WAL) and dstate.RemoveFiles, and use them for the migration's temp state. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent ef609ea commit 1941ff7

3 files changed

Lines changed: 46 additions & 6 deletions

File tree

‎bundle/direct/dstate/state.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,26 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported
833833
return state, err
834834
}
835835

836+
// Discard closes the state without persisting it and removes its file and WAL. It is for
837+
// temporary states, such as a migration's, that must not outlive a failure: the open WAL handle
838+
// is closed first, without which the removal fails on Windows.
839+
func (db *DeploymentState) Discard() {
840+
db.mu.Lock()
841+
path := db.Path
842+
db.reset()
843+
db.mu.Unlock()
844+
845+
if path != "" {
846+
RemoveFiles(path)
847+
}
848+
}
849+
850+
// RemoveFiles removes the state file at path and its WAL. Missing files are ignored.
851+
func RemoveFiles(path string) {
852+
_ = os.Remove(path)
853+
_ = os.Remove(path + walSuffix)
854+
}
855+
836856
// UpgradeToWrite transitions from read mode to write mode without re-reading state.
837857
// State must already be open for read. This initializes the WAL for writing.
838858
func (db *DeploymentState) UpgradeToWrite() error {

‎bundle/direct/dstate/state_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,27 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) {
4545
assert.ErrorIs(t, err, os.ErrNotExist)
4646
}
4747

48+
func TestDiscardRemovesStateAndWAL(t *testing.T) {
49+
path := filepath.Join(t.TempDir(), "state.json")
50+
51+
var db DeploymentState
52+
db.OpenWithData(path, NewDatabase("lineage", 1))
53+
require.NoError(t, db.UpgradeToWrite())
54+
require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil))
55+
db.Discard()
56+
57+
_, err := os.Stat(path)
58+
assert.ErrorIs(t, err, os.ErrNotExist)
59+
_, err = os.Stat(path + walSuffix)
60+
assert.ErrorIs(t, err, os.ErrNotExist)
61+
62+
// The WAL is opened with O_EXCL, so a leftover one would fail a fresh write at the same path.
63+
var db2 DeploymentState
64+
db2.OpenWithData(path, NewDatabase("lineage", 1))
65+
require.NoError(t, db2.UpgradeToWrite())
66+
db2.Discard()
67+
}
68+
4869
func TestExportStateFromDataJobRunJobID(t *testing.T) {
4970
data := Database{
5071
State: map[string]ResourceEntry{

‎bundle/statemgmt/direct_migration.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) {
6767
if tempStatePath != "" {
6868
// Always remove the temp state and its WAL. The converted state is loaded into memory
6969
// below and never renamed into place, so these are the only on-disk copies to clean up.
70-
defer func() {
71-
_ = os.Remove(tempStatePath)
72-
_ = os.Remove(tempStatePath + ".wal")
73-
}()
70+
defer dstate.RemoveFiles(tempStatePath)
7471
}
7572
if hasWarnings {
7673
b.Metrics.SetBoolValue(metrics.DirectMigrateWarnings, true)
@@ -233,8 +230,7 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr
233230
tempStatePath := filepath.Join(filepath.Dir(localDirectPath), "resources.migrating.json")
234231
// Clean up any leftovers from a crashed previous run so UpgradeToWrite
235232
// (which opens the .wal with O_EXCL) succeeds.
236-
_ = os.Remove(tempStatePath)
237-
_ = os.Remove(tempStatePath + ".wal")
233+
dstate.RemoveFiles(tempStatePath)
238234

239235
// SecretScopeFixups and the direct-engine state builder report failures via
240236
// logdiag. Run them in an isolated + collecting context so their diagnostics
@@ -258,6 +254,9 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr
258254
// stays at tf+1. Either way it outranks the terraform state, which is all engine selection needs.
259255
var stateDB dstate.DeploymentState
260256
stateDB.OpenWithData(tempStatePath, dstate.NewDatabase(tfState.Lineage, tfState.Serial+1))
257+
// On a failed conversion, close the WAL so the caller can remove the files (Windows refuses to
258+
// remove an open file). After the successful Finalize below this is a no-op.
259+
defer stateDB.Discard()
261260

262261
// Apply SecretScopeFixups so the config matches what the direct engine expects.
263262
// This adds MANAGE ACL for the current user to all secret scopes, ensuring

0 commit comments

Comments
 (0)