Skip to content

Commit 88152b3

Browse files
denikIsaac
denik
and
Isaac
authored
acc: move fs cp to non-existent directory test to acceptance; add DBFS to testserver (#7251)
## Changes - Add `acceptance/cmd/fs/cp/file-to-nonexistent-dir` (`Cloud = true`) and delete `integration/cmd/fs.TestFsCpFileToNonExistentDir`. The test copies a file with a trailing-slash target into an existing directory (succeeds) and into a missing one (`no such directory`), for every combination of local, DBFS and UC Volume source and target — the same matrix as the integration test. - Add the DBFS API to the testserver (`libs/testserver/dbfs.go`): `get-status`, `list`, `mkdirs`, `create`/`add-block`/`close`, `read` and `delete`, in a namespace separate from workspace files, with the status and error codes the DBFS filer maps (`RESOURCE_DOES_NOT_EXIST`, `RESOURCE_ALREADY_EXISTS`, `IO_ERROR` for a non-empty directory). ## Why `integration/` is deprecated (`.agents/rules/testing.md`), and the acceptance version also runs locally against the testserver. Dashboard: test [integration/cmd/fs.TestFsCpFileToNonExistentDir/uc-volumes_to_uc-volumes](http://go/cli/dashboard/test/integration/cmd/fs.TestFsCpFileToNonExistentDir/uc-volumes_to_uc-volumes?main=1&pr=1&end=2026-11-03T00:00:00Z&last=33d) (the test being moved). ## Tests - New `acceptance/cmd/fs/dbfs` (`Cloud = true`) runs `fs` commands against DBFS: a 1.5 MB file round trip (several DBFS blocks), already-exists without `--overwrite`, missing parent, `ls` of a file, `cat` of a directory, non-empty and recursive `rm`. - Both acceptance tests pass locally and against a real AWS workspace (`testme-aws`, 0 skipped), so the testserver's DBFS matches the real API on these paths. This pull request and its description were written by Isaac. --------- Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 050b89d commit 88152b3

14 files changed

Lines changed: 644 additions & 31 deletions

File tree

‎acceptance/cmd/fs/cp/file-to-nonexistent-dir/foo.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
abc

‎acceptance/cmd/fs/cp/file-to-nonexistent-dir/out.test.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Cloud = true

‎acceptance/cmd/fs/cp/file-to-nonexistent-dir/output.txt‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
2+
=== foo.txt -> local
3+
>>> [CLI] fs cp foo.txt local/existing-from-local/
4+
foo.txt -> local/existing-from-local/foo.txt
5+
6+
>>> [CLI] fs cp foo.txt local/nonexistent/mydir/
7+
Error: no such directory: local/nonexistent/mydir
8+
9+
Exit code: 1
10+
11+
=== foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]
12+
>>> [CLI] fs cp foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-local/
13+
foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-local/foo.txt
14+
15+
>>> [CLI] fs cp foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent/mydir/
16+
Error: no such directory: /tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent
17+
18+
Exit code: 1
19+
20+
=== foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]
21+
>>> [CLI] fs cp foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-local/
22+
foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-local/foo.txt
23+
24+
>>> [CLI] fs cp foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexistent/mydir/
25+
Error: no such directory: /Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexistent
26+
27+
Exit code: 1
28+
29+
=== dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local
30+
>>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt local/existing-from-dbfs/
31+
dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local/existing-from-dbfs/foo.txt
32+
33+
>>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt local/nonexistent/mydir/
34+
Error: no such directory: local/nonexistent/mydir
35+
36+
Exit code: 1
37+
38+
=== dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]
39+
>>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/
40+
dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/foo.txt
41+
42+
>>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent/mydir/
43+
Error: no such directory: /tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent
44+
45+
Exit code: 1
46+
47+
=== dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]
48+
>>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/
49+
dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/foo.txt
50+
51+
>>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexistent/mydir/
52+
Error: no such directory: /Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexistent
53+
54+
Exit code: 1
55+
56+
=== dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local
57+
>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt local/existing-from-volume/
58+
dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local/existing-from-volume/foo.txt
59+
60+
>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt local/nonexistent/mydir/
61+
Error: no such directory: local/nonexistent/mydir
62+
63+
Exit code: 1
64+
65+
=== dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]
66+
>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/
67+
dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/foo.txt
68+
69+
>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent/mydir/
70+
Error: no such directory: /tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent
71+
72+
Exit code: 1
73+
74+
=== dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]
75+
>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/
76+
dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/foo.txt
77+
78+
>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexistent/mydir/
79+
Error: no such directory: /Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexistent
80+
81+
Exit code: 1
82+
83+
=== Copies landed in the existing directories
84+
>>> cat local/existing-from-local/foo.txt
85+
abc
86+
87+
>>> [CLI] fs cat dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-local/foo.txt
88+
abc
89+
90+
>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-local/foo.txt
91+
abc
92+
93+
>>> cat local/existing-from-dbfs/foo.txt
94+
abc
95+
96+
>>> [CLI] fs cat dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/foo.txt
97+
abc
98+
99+
>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/foo.txt
100+
abc
101+
102+
>>> cat local/existing-from-volume/foo.txt
103+
abc
104+
105+
>>> [CLI] fs cat dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/foo.txt
106+
abc
107+
108+
>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/foo.txt
109+
abc

‎acceptance/cmd/fs/cp/file-to-nonexistent-dir/script‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# A trailing slash on the target means "copy into this directory": it works when the directory
2+
# exists and fails with "no such directory" when it does not. Covers local, DBFS and UC Volume
3+
# sources and targets.
4+
CATALOG_NAME="main"
5+
SCHEMA_NAME="default"
6+
VOLUME_NAME="fs-cp-test-${UNIQUE_NAME}"
7+
VOLUME="dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}"
8+
DBFS="dbfs:/tmp/fs-cp-test-${UNIQUE_NAME}"
9+
10+
cleanup() {
11+
$CLI volumes delete "${CATALOG_NAME}.${SCHEMA_NAME}.${VOLUME_NAME}" 2>/dev/null || true
12+
$CLI fs rm -r ${DBFS} 2>/dev/null || true
13+
}
14+
trap cleanup EXIT
15+
16+
$CLI volumes create "${CATALOG_NAME}" "${SCHEMA_NAME}" "${VOLUME_NAME}" MANAGED >/dev/null
17+
for root in ${VOLUME} ${DBFS}; do
18+
$CLI fs mkdir $root/src
19+
$CLI fs cp foo.txt $root/src/foo.txt >/dev/null
20+
done
21+
22+
for src_kind in local dbfs volume; do
23+
case $src_kind in
24+
local) src=foo.txt ;;
25+
dbfs) src=${DBFS}/src/foo.txt ;;
26+
volume) src=${VOLUME}/src/foo.txt ;;
27+
esac
28+
# Separate existing directory per source, so later copies are not skipped as already existing.
29+
existing="existing-from-$src_kind"
30+
mkdir -p local/$existing
31+
$CLI fs mkdir ${DBFS}/$existing
32+
$CLI fs mkdir ${VOLUME}/$existing
33+
for dst in local ${DBFS} ${VOLUME}; do
34+
title "$src -> $dst"
35+
trace $CLI fs cp $src $dst/$existing/
36+
errcode trace $CLI fs cp $src $dst/nonexistent/mydir/
37+
done
38+
done
39+
40+
title "Copies landed in the existing directories"
41+
for existing in existing-from-local existing-from-dbfs existing-from-volume; do
42+
trace cat local/$existing/foo.txt
43+
trace $CLI fs cat ${DBFS}/$existing/foo.txt
44+
trace $CLI fs cat ${VOLUME}/$existing/foo.txt
45+
done
46+
rm -rf local

‎acceptance/cmd/fs/cp/file-to-nonexistent-dir/test.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Cloud = true
2+
3+
# Local paths in errors use the OS separator.
4+
[[Repls]]
5+
Old = 'local\\nonexistent\\mydir'
6+
New = 'local/nonexistent/mydir'

‎acceptance/cmd/fs/dbfs/out.test.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Cloud = true

‎acceptance/cmd/fs/dbfs/output.txt‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
2+
=== Create a directory and write a file in it
3+
>>> [CLI] fs mkdir dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir
4+
5+
>>> [CLI] fs cp big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
6+
big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
7+
8+
=== Read the file back
9+
>>> cmp big.bin big.back
10+
contents match
11+
12+
=== Writing over an existing file needs --overwrite
13+
>>> [CLI] fs cp big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
14+
big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin (skipped; already exists)
15+
16+
>>> [CLI] fs cp --overwrite big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
17+
big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
18+
19+
=== Writing into a directory that does not exist fails
20+
>>> [CLI] fs cp big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/missing/big.bin
21+
Error: no such directory: /tmp/fs-dbfs-[UNIQUE_NAME]/missing
22+
23+
Exit code: 1
24+
25+
=== List a directory, then a file
26+
>>> [CLI] fs ls dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir
27+
big.bin
28+
29+
>>> [CLI] fs ls dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
30+
Error: not a directory: /tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin
31+
32+
Exit code: 1
33+
34+
=== cat a directory fails
35+
>>> [CLI] fs cat dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir
36+
Error: not a file: /tmp/fs-dbfs-[UNIQUE_NAME]/dir
37+
38+
Exit code: 1
39+
40+
=== rm a non-empty directory needs -r
41+
>>> [CLI] fs rm dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir
42+
Error: directory not empty: /tmp/fs-dbfs-[UNIQUE_NAME]/dir
43+
44+
Exit code: 1
45+
46+
>>> [CLI] fs rm -r dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir
47+
48+
>>> [CLI] fs ls dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir
49+
Error: no such directory: /tmp/fs-dbfs-[UNIQUE_NAME]/dir
50+
51+
Exit code: 1

‎acceptance/cmd/fs/dbfs/script‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# fs commands against DBFS: the filer's error mapping for DBFS-specific responses, and a file over
2+
# the 1 MB DBFS block size so writes and reads are split into several API calls.
3+
DBFS="dbfs:/tmp/fs-dbfs-${UNIQUE_NAME}"
4+
5+
cleanup() {
6+
$CLI fs rm -r ${DBFS} 2>/dev/null || true
7+
rm -f big.bin big.back
8+
}
9+
trap cleanup EXIT
10+
11+
python3 -c 'import sys; sys.stdout.buffer.write(bytes(range(256)) * 6000)' > big.bin
12+
13+
title "Create a directory and write a file in it"
14+
trace $CLI fs mkdir ${DBFS}/dir
15+
trace $CLI fs cp big.bin ${DBFS}/dir/big.bin
16+
17+
title "Read the file back"
18+
$CLI fs cp ${DBFS}/dir/big.bin big.back > /dev/null
19+
trace cmp big.bin big.back && echo "contents match"
20+
21+
title "Writing over an existing file needs --overwrite"
22+
trace $CLI fs cp big.bin ${DBFS}/dir/big.bin
23+
trace $CLI fs cp --overwrite big.bin ${DBFS}/dir/big.bin
24+
25+
title "Writing into a directory that does not exist fails"
26+
errcode trace $CLI fs cp big.bin ${DBFS}/missing/big.bin
27+
28+
title "List a directory, then a file"
29+
trace $CLI fs ls ${DBFS}/dir
30+
errcode trace $CLI fs ls ${DBFS}/dir/big.bin
31+
32+
title "cat a directory fails"
33+
errcode trace $CLI fs cat ${DBFS}/dir
34+
35+
title "rm a non-empty directory needs -r"
36+
errcode trace $CLI fs rm ${DBFS}/dir
37+
trace $CLI fs rm -r ${DBFS}/dir
38+
errcode trace $CLI fs ls ${DBFS}/dir

‎acceptance/cmd/fs/dbfs/test.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Cloud = true

‎integration/cmd/fs/cp_test.go‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -369,37 +369,6 @@ func TestFsCpSourceIsDirectoryButTargetIsFile(t *testing.T) {
369369
}
370370
}
371371

372-
func TestFsCpFileToNonExistentDir(t *testing.T) {
373-
t.Parallel()
374-
375-
for _, testCase := range copyTests() {
376-
t.Run(testCase.name, func(t *testing.T) {
377-
t.Parallel()
378-
379-
ctx := t.Context()
380-
sourceFiler, sourceDir := testCase.setupSource(t)
381-
targetFiler, targetDir := testCase.setupTarget(t)
382-
setupSourceFile(t, ctx, sourceFiler)
383-
384-
// Create a directory in the target
385-
err := targetFiler.Mkdir(ctx, "existingdir")
386-
require.NoError(t, err)
387-
388-
// Copy file to existing directory with trailing slash - should succeed
389-
existingDir := path.Join(targetDir, "existingdir") + "/"
390-
testcli.RequireSuccessfulRun(t, ctx, "fs", "cp", path.Join(sourceDir, "foo.txt"), existingDir)
391-
assertTargetFile(t, ctx, targetFiler, "existingdir/foo.txt")
392-
393-
// Try to copy file to a non-existent directory with trailing slash - should fail
394-
nonExistentDir := path.Join(targetDir, "nonexistent", "mydir") + "/"
395-
_, _, err = testcli.RequireErrorRun(t, ctx, "fs", "cp", path.Join(sourceDir, "foo.txt"), nonExistentDir)
396-
assert.Error(t, err)
397-
assert.Contains(t, err.Error(), "directory")
398-
assert.Contains(t, err.Error(), "no such directory")
399-
})
400-
}
401-
}
402-
403372
func TestFsCpFileToNonExistentDirWindowsPaths(t *testing.T) {
404373
if runtime.GOOS != "windows" {
405374
t.Skip("Skipping test on non-windows OS")

0 commit comments

Comments
 (0)