diff --git a/.nextchanges/cli/sandbox-ssh-port-forward.md b/.nextchanges/cli/sandbox-ssh-port-forward.md new file mode 100644 index 00000000000..5dfbf4bfe97 --- /dev/null +++ b/.nextchanges/cli/sandbox-ssh-port-forward.md @@ -0,0 +1 @@ +* Use local SSH keys and authenticated WebSocket port forwarding for `databricks sandbox ssh`, adding SSH support for sandboxes created through the public Databricks SDK or API. The command no longer accepts `--port`. The sandbox environment must have Python 3 and OpenSSH server (`openssh-server`) installed. ([#7589](https://github.com/databricks/cli/pull/7589)) diff --git a/NOTICE b/NOTICE index fb51e5c0914..72fd47a044d 100644 --- a/NOTICE +++ b/NOTICE @@ -4,6 +4,10 @@ This Software includes software developed at Databricks (https://www.databricks. This Software contains code from the following open source projects, licensed under the Apache 2.0 license: +grpc/grpc-go - https://github.com/grpc/grpc-go +Copyright The gRPC Authors +License - https://github.com/grpc/grpc-go/blob/master/LICENSE + spf13/cobra - https://github.com/spf13/cobra Copyright cobra authors License - https://github.com/spf13/cobra/blob/main/LICENSE.txt @@ -33,6 +37,10 @@ License - https://github.com/gorilla/websocket/blob/main/LICENSE This Software contains code from the following open source projects, licensed under the BSD (3-clause) license: +protocolbuffers/protobuf-go - https://github.com/protocolbuffers/protobuf-go +Copyright The Go Authors +License - https://github.com/protocolbuffers/protobuf-go/blob/master/LICENSE + spf13/pflag - https://github.com/spf13/pflag Copyright (c) 2012 Alex Ogier. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. diff --git a/cmd/cmd.go b/cmd/cmd.go index e1cf2fdabef..9d656137bd1 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -96,7 +96,7 @@ func sandboxSpec() root.CommandSpec { spec := sandbox.Spec spec.Build = func(ctx context.Context, cmd *cobra.Command) { sandbox.Spec.Build(ctx, cmd) - cmd.AddCommand(sandboxssh.NewServerCommand()) + cmd.AddCommand(sandboxssh.NewServerCommand(), sandboxssh.NewSSHCommand(sandboxssh.CommandDependencies{ResolveName: sandbox.ResolveSSHName, Complete: sandbox.CompleteSandboxIDs})) } return spec } diff --git a/cmd/sandbox/completion.go b/cmd/sandbox/completion.go index e7a74ab2472..12b38b0c9b3 100644 --- a/cmd/sandbox/completion.go +++ b/cmd/sandbox/completion.go @@ -66,3 +66,8 @@ func completionProfile(cmd *cobra.Command, args []string) string { } return cacheProfile(cmdctx.WorkspaceClient(cmd.Context())) } + +// CompleteSandboxIDs completes sandbox selections without making API requests. +func CompleteSandboxIDs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return completeSandboxIDs(cmd, args, toComplete) +} diff --git a/cmd/sandbox/sandbox.go b/cmd/sandbox/sandbox.go index 9d7754ea457..f605113bddc 100644 --- a/cmd/sandbox/sandbox.go +++ b/cmd/sandbox/sandbox.go @@ -24,7 +24,6 @@ tooling (Python, Node.js, Rust, Databricks CLI) and persistent storage. Getting started: databricks auth login --host https://... # authenticate to a Databricks workspace - databricks sandbox register # generate and register an SSH key databricks sandbox create # provision a sandbox (becomes your default) databricks sandbox ssh # SSH to your default sandbox @@ -41,7 +40,6 @@ Common workflows: cmd.AddCommand(newRegisterCommand()) cmd.AddCommand(newSSHKeyCommand()) cmd.AddCommand(newSetDefaultCommand()) - cmd.AddCommand(newSSHCommand()) cmd.AddCommand(newForwardCommand()) cmd.AddCommand(newListCommand()) cmd.AddCommand(newCreateCommand()) diff --git a/cmd/sandbox/ssh.go b/cmd/sandbox/ssh.go index 711bbd8b9a4..1904005c85d 100644 --- a/cmd/sandbox/ssh.go +++ b/cmd/sandbox/ssh.go @@ -4,268 +4,43 @@ import ( "context" "errors" "fmt" - "io/fs" - "os" "strings" + "unicode" - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/execv" - "github.com/databricks/cli/libs/shellquote" - "github.com/spf13/cobra" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + sdk "github.com/databricks/databricks-sdk-go/service/sandbox" ) -const defaultGatewayPort = "2222" - -func newSSHCommand() *cobra.Command { - var gatewayPort string - - cmd := &cobra.Command{ - Use: "ssh [sandbox-id] [-- ...]", - Short: "SSH into a Sandbox environment", - Long: `SSH into a Sandbox environment. - -Connect to your default or a named sandbox via SSH. Extra arguments -after -- are passed directly to the ssh process. This lets you run -remote commands, set up port forwarding, or pass any other ssh flags. - -Examples: - databricks sandbox ssh # interactive shell on default sandbox - databricks sandbox ssh happy-panda-1234 # interactive shell on specific sandbox - databricks sandbox ssh -- ls -la /home # run command on default sandbox - databricks sandbox ssh happy-panda-1234 -- cat /etc/os-release # run command on specific sandbox - databricks sandbox ssh -- -L 8080:localhost:8080 # port forwarding on default sandbox`, - Args: cobra.ArbitraryArgs, - PreRunE: root.MustWorkspaceClient, - ValidArgsFunction: completeSandboxIDs, - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - profile := w.Config.Profile - if profile == "" { - profile = w.Config.Host - } - - keyPath, err := sandboxKeyPath(ctx) - if err != nil { - return fmt.Errorf("failed to determine sandbox key path: %w", err) - } - if _, err := os.Stat(keyPath); errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("sandbox SSH key not found at %s — run 'databricks sandbox register' first", keyPath) - } - - var sandboxID string - var extraArgs []string - - switch dashAt := cmd.ArgsLenAtDash(); dashAt { - case -1: - if len(args) > 0 { - sandboxID = args[0] - } - case 0: - extraArgs = args[dashAt:] - default: - sandboxID = args[0] - extraArgs = args[dashAt:] - } - - api, err := newSandboxAPI(w) - if err != nil { - return err - } - - // Surface deleted-key errors before ssh's opaque "Permission denied". - if err := verifyKeyRegistered(ctx, api, keyPath); err != nil { - return err - } - - // Validating the sandbox against the server surfaces an unknown ID - // as a clear 404 instead of ssh's `Permission denied` when the - // gateway can't route it, and captures `gateway_host` for the - // resolution below. - sandboxID, sb, err := resolveSandbox(ctx, api, profile, sandboxID) - if err != nil { - return err - } - // "" when the lookup only warned. - var sandboxGatewayHost, sandboxStatus string - if sb != nil { - sandboxGatewayHost = sb.GatewayHost - sandboxStatus = sb.Status - } - - // Drive start ourselves so the user sees a spinner instead - // of an opaque multi-minute hang inside ssh's connect path. - if sandboxStatus != "" && !strings.EqualFold(sandboxStatus, "running") { - final, err := ensureRunning(ctx, api, sandboxID, sandboxStatus) - if err != nil { - return err - } - if final.GatewayHost != "" { - sandboxGatewayHost = final.GatewayHost - } - } - - // Resolution precedence: fresh API response → cached value - // for this profile. The server stamps the workspace's - // gateway on every Sandbox / SshKey response, so the cache - // is populated after the first call against the workspace - // (sandbox register / list / create / etc.). - host := sandboxGatewayHost - if host == "" { - host = getGatewayHost(ctx, profile) - } - if host == "" { - return errors.New("could not resolve the sandbox SSH gateway — run `databricks sandbox register` first") - } - - // Persist whatever the server just told us, so the next invocation - // can short-circuit the explicit-id `get` above. - if sandboxGatewayHost != "" { - _ = setGatewayHost(ctx, profile, sandboxGatewayHost) - } - - // No "Connecting…" spinner here. execSSHDirect replaces - // the CLI process via execv, so a deferred Close() would - // never run — and Bubble Tea hides the terminal cursor at - // init and only restores it on clean shutdown, leaving the - // cursor invisible for the entire ssh session and after it - // exits. The spinner couldn't animate during the handshake - // anyway (the CLI process is gone by then). - return execSSHDirect(sandboxID, host, gatewayPort, keyPath, extraArgs) - }, +// ResolveSSHName resolves the sandbox selection shared by SSH commands. +func ResolveSSHName(ctx context.Context, w *databricks.WorkspaceClient, requested string) (string, error) { + profile := w.Config.Profile + if profile == "" { + profile = w.Config.Host } - - cmd.Flags().StringVar(&gatewayPort, "port", defaultGatewayPort, "Sandbox gateway SSH port") - - return cmd -} - -// verifyKeyRegistered confirms the local sandbox public key is still -// registered with the workspace before we open the SSH socket. The -// gateway itself already does this check during userauth, but the -// SSH protocol's reply surface (USERAUTH_FAILURE has no free-form -// reason; USERAUTH_BANNER is widely swallowed) flattens "unknown -// key", "key registered but not authorized for this sandbox", and -// "upstream service is down" into the same "Permission denied -// (publickey)". This out-of-band HTTP check surfaces the specific -// case in language the user can act on. listKeys errors fall -// through with a warning so a transient API hiccup doesn't block a -// connection the gateway could still route. -func verifyKeyRegistered(ctx context.Context, api *sandboxAPI, keyPath string) error { - pub, err := os.ReadFile(keyPath + ".pub") - if err != nil { - return fmt.Errorf("reading public key %s.pub: %w", keyPath, err) + savedDefault := requested == "" + if savedDefault { + requested = getDefault(ctx, profile) } - want := keyHash(string(pub)) - - keys, err := api.listKeys(ctx) - if err != nil { - warn(ctx, fmt.Sprintf("Could not verify SSH key registration: %v", err)) - return nil - } - for _, k := range keys { - if k.KeyHash == want { - return nil - } + if requested == "" { + return "", errors.New("no default sandbox configured — specify a sandbox ID or run 'databricks sandbox default '") } - return fmt.Errorf("your sandbox SSH key (%s) is not registered with this workspace — run `databricks sandbox register` to re-register it", want) -} - -// ensureRunning owns the lifecycle transition so SSH does not hang inside -// connection setup. The caller must not have a spinner open. -func ensureRunning(ctx context.Context, api *sandboxAPI, id, currentStatus string) (*sandboxEntry, error) { - if strings.EqualFold(currentStatus, "running") { - return &sandboxEntry{SandboxID: id, Status: currentStatus}, nil + id, err := resolveLocalID(ctx, profile, strings.TrimPrefix(requested, "sandboxes/")) + if err != nil { + return "", err } - - s := spin(ctx, "Starting "+cmdio.Bold(ctx, id)+"…") - defer s.Close() - - var sb *sandboxEntry - if strings.EqualFold(currentStatus, "stopping") { - stopped, err := waitForStopped(ctx, api, s, id) - if err != nil { - s.fail("Failed to start " + id) - return nil, err - } - currentStatus = stopped.Status - s.Update("Starting " + id + "…") + if err := validateSandboxID(id); err != nil { + return "", err } - - if strings.EqualFold(currentStatus, "stopped") { - updated, err := api.start(ctx, id) - if err != nil { - s.fail("Failed to start " + id) - return nil, fmt.Errorf("failed to start sandbox %s: %w", id, err) - } - sb = updated + if id == "" || strings.ContainsAny(id, "?#") || strings.ContainsFunc(id, unicode.IsSpace) { + return "", fmt.Errorf("invalid sandbox ID %q", id) } - - if sb == nil || !strings.EqualFold(sb.Status, "running") { - final, err := waitForRunning(ctx, api, s, id) - if err != nil { - s.fail("Failed to start " + id) - return nil, err + name := "sandboxes/" + id + if _, err := w.Sandbox.GetSandbox(ctx, sdk.GetSandboxRequest{Name: name}); err != nil { + if savedDefault && errors.Is(err, apierr.ErrNotFound) { + return "", fmt.Errorf("saved default %q is unavailable through the Sandbox API; verify the profile or choose another sandbox: %w", id, err) } - sb = final - } - - s.ok("Started " + cmdio.Bold(ctx, id)) - return sb, nil -} - -// execSSHDirect replaces the CLI process with ssh (or simulates that on -// Windows via execv). All options are passed on the command line, so no -// ~/.ssh/config entry is required. -func execSSHDirect(sandboxID, host, port, keyPath string, extraArgs []string) error { - return execv.Execv(execv.Options{ - Args: buildSSHArgs(sandboxID, host, port, keyPath, extraArgs), - Env: os.Environ(), - }) -} - -// buildSSHArgs assembles the argv we hand to the `ssh` binary. -// -// `ssh` concatenates remote-command words with spaces and the remote -// shell re-parses them. That makes the two natural user shapes -// incompatible by default: -// -// - Single arg that's already a complete shell command: -// `sandbox ssh -- 'echo hi | head -3'` -// The user expects the remote shell to split and execute this -// string. ssh's "concat with spaces" is a no-op here, so we hand -// the arg through untouched. -// -// - Multi-arg exec-style invocation: -// `sandbox ssh -- bash -c 'echo hi'` -// Cobra splits this into `["bash", "-c", "echo hi"]`. ssh's join -// produces `bash -c echo hi` on the wire, which bash re-splits into -// `-c=echo` and `$0=hi` — losing the second word. We fix that by shell-quoting -// each arg before append, so the remote sees `bash -c 'echo hi'`. -// -// The heuristic: if there's exactly one extra arg, pass it untouched; -// otherwise quote every arg. shellquote.BashArg leaves safe args alone, -// so `ls -la /tmp` round-trips unchanged in the multi-arg path. -func buildSSHArgs(sandboxID, host, port, keyPath string, extraArgs []string) []string { - args := []string{ - "ssh", - "-i", keyPath, - "-p", port, - "-o", "IdentitiesOnly=yes", - "-o", "PreferredAuthentications=publickey", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - fmt.Sprintf("%s@%s", sandboxID, host), - } - if len(extraArgs) == 1 { - return append(args, extraArgs[0]) - } - for _, a := range extraArgs { - args = append(args, shellquote.BashArg(a)) + return "", err } - return args + return name, nil } diff --git a/cmd/sandbox/ssh_test.go b/cmd/sandbox/ssh_test.go index 31f53eee6b4..60409f2f4ce 100644 --- a/cmd/sandbox/ssh_test.go +++ b/cmd/sandbox/ssh_test.go @@ -1,215 +1,47 @@ package sandbox import ( - "encoding/json" + "io" "net/http" "net/http/httptest" - "sync/atomic" + "strings" "testing" - "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestEnsureRunningLifecycle(t *testing.T) { - type requestStep struct { - method string - path string - status string - } - - for _, tc := range []struct { - name string - initialStatus string - steps []requestStep - wantStatus string - wantErr string - wantStarts int32 - }{ - { - name: "stopping waits for stopped before starting", - initialStatus: "Stopping", - steps: []requestStep{ - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Stopped"}, - {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Running"}, - }, - wantStatus: "Running", - wantStarts: 1, - }, - { - name: "stopped starts normally", - initialStatus: "Stopped", - steps: []requestStep{ - {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Running"}, - }, - wantStatus: "Running", - wantStarts: 1, - }, - { - name: "running is a no-op", - initialStatus: "Running", - wantStatus: "Running", - }, - { - name: "stopped after start remains an error", - initialStatus: "Stopped", - steps: []requestStep{ - {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Stopped"}, - }, - wantErr: `sandbox test-id reached unexpected state "Stopped" while starting`, - wantStarts: 1, - }, - } { - t.Run(tc.name, func(t *testing.T) { - var requestCount atomic.Int32 - var startCount atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/.well-known/databricks-config" { - assert.NoError(t, json.NewEncoder(w).Encode(map[string]string{ - "oidc_endpoint": "https://workspace.example.test/oidc", - })) - return - } - - index := int(requestCount.Add(1) - 1) - if index >= len(tc.steps) { - t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) - http.Error(w, "unexpected request", http.StatusInternalServerError) - return - } - - step := tc.steps[index] - assert.Equal(t, step.method, r.Method) - assert.Equal(t, step.path, r.URL.Path) - if r.Method == http.MethodPost && r.URL.Path == sandboxPath("test-id")+"/start" { - startCount.Add(1) - } - assert.NoError(t, json.NewEncoder(w).Encode(sandboxEntry{ - SandboxID: "test-id", - Status: step.status, - })) - })) - t.Cleanup(server.Close) - - workspaceClient, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "test-token", - }) - require.NoError(t, err) - api, err := newSandboxAPI(workspaceClient) - require.NoError(t, err) - - ctx := cmdio.MockDiscard(t.Context()) - got, err := ensureRunning(ctx, api, "test-id", tc.initialStatus) - if tc.wantErr != "" { - require.EqualError(t, err, tc.wantErr) - assert.Nil(t, got) - } else { - require.NoError(t, err) - require.NotNil(t, got) - assert.Equal(t, tc.wantStatus, got.Status) - } - assert.Equal(t, int32(len(tc.steps)), requestCount.Load()) - assert.Equal(t, tc.wantStarts, startCount.Load()) - }) - } -} - -func TestBuildSSHArgsBaseFlags(t *testing.T) { - got := buildSSHArgs("happy-panda-1234", "gw.example.test", "2222", "/keys/id", nil) - // Sanity: the destination is the last arg when no extras. - assert.Equal(t, "happy-panda-1234@gw.example.test", got[len(got)-1]) - // The base flag set is fixed; spot-check a couple. - assert.Contains(t, got, "-i") - assert.Contains(t, got, "/keys/id") - assert.Contains(t, got, "-p") - assert.Contains(t, got, "2222") -} - -func TestBuildSSHArgsQuoting(t *testing.T) { - for _, tc := range []struct { - name string - extraArgs []string - // expected is what we expect at the end of args, after the destination. - expected []string - }{ - { - name: "no extras", - extraArgs: nil, - expected: nil, - }, - { - name: "single safe word — passed through", - extraArgs: []string{"uname"}, - expected: []string{"uname"}, - }, - { - name: "single string with spaces — passed through so the remote shell can split it", - extraArgs: []string{"echo hello-from-string"}, - expected: []string{"echo hello-from-string"}, - }, - { - name: "single string with a pipe — passed through to the remote shell", - extraArgs: []string{"cat /etc/os-release | head -3"}, - expected: []string{"cat /etc/os-release | head -3"}, - }, - { - name: "multi safe words — no quoting", - extraArgs: []string{"ls", "-la", "/tmp"}, - expected: []string{"ls", "-la", "/tmp"}, - }, - { - name: "bash -c '' — third arg gets quoted", - // The whole point: without the quoting, the remote shell - // re-splits "bash -c echo hi" and bash's -c eats just "echo". - extraArgs: []string{"bash", "-c", "echo hi"}, - expected: []string{"bash", "-c", "'echo hi'"}, - }, - { - name: "arg with single quote — escaped, not lost", - extraArgs: []string{"echo", "it's fine"}, - expected: []string{"echo", `'it'\''s fine'`}, - }, - { - name: "glob pattern — quoted so the remote (not the local) shell expands it", - extraArgs: []string{"find", ".", "-name", "*.txt"}, - expected: []string{"find", ".", "-name", "'*.txt'"}, - }, - { - name: "ssh -o key=val style — gets quoted because of '='; harmless given current arg ordering", - extraArgs: []string{"-o", "StrictHostKeyChecking=no"}, - expected: []string{"-o", "'StrictHostKeyChecking=no'"}, - }, - { - name: "empty arg gets explicit empty quotes (not lost)", - extraArgs: []string{"sh", "-c", ""}, - expected: []string{"sh", "-c", "''"}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - got := buildSSHArgs("id", "host", "2222", "/k", tc.extraArgs) - // Trim everything up to and including the destination so we - // can assert just on the appended extras. - dest := "id@host" - cut := -1 - for i, a := range got { - if a == dest { - cut = i + 1 - break - } - } - assert.GreaterOrEqual(t, cut, 0, "destination not found in args") - tail := got[cut:] - if tc.expected == nil { - assert.Empty(t, tail) - } else { - assert.Equal(t, tc.expected, tail) - } - }) +func TestResolveSSHName(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + calls++ + if strings.HasSuffix(r.URL.Path, "missing") { + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"error_code":"NOT_FOUND","message":"missing"}`) + return + } + _, _ = io.WriteString(w, `{"name":"sandboxes/test","status":{"state":"SANDBOX_STATE_RUNNING"}}`) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + require.NoError(t, setDefault(ctx, server.URL, "missing")) + _, err = ResolveSSHName(ctx, w, "") + require.ErrorContains(t, err, "unavailable through the Sandbox API") + assert.Equal(t, "missing", getDefault(ctx, server.URL)) + for _, id := range []string{".", "..", "sandboxes/.", "sandboxes/..", "a/b", `a\b`, "a?b", "a#b", "a b", "a\tb"} { + _, err = ResolveSSHName(ctx, w, id) + require.ErrorContains(t, err, "invalid sandbox ID", id) } + name, err := ResolveSSHName(ctx, w, "sandboxes/test") + require.NoError(t, err) + assert.Equal(t, "sandboxes/test", name) + assert.Equal(t, 2, calls) } diff --git a/cmd/sandbox/sshconfig.go b/cmd/sandbox/sshconfig.go index d4c54986aed..ac98d0ed70a 100644 --- a/cmd/sandbox/sshconfig.go +++ b/cmd/sandbox/sshconfig.go @@ -17,6 +17,7 @@ import ( ) const ( + defaultGatewayPort = "2222" // sshIncludeFileName is the sandbox-managed file referenced by the // user's ~/.ssh/config. The file is fully owned: we rewrite it on // every `sandbox register`, so manual edits to it will not survive. diff --git a/cmd/sandbox/start.go b/cmd/sandbox/start.go index ba48a42efd9..9624a7aab1d 100644 --- a/cmd/sandbox/start.go +++ b/cmd/sandbox/start.go @@ -104,7 +104,7 @@ func waitForStopped(ctx context.Context, api *sandboxAPI, s *spinner, id string) } // Centralizing lifecycle polling keeps timeout, cancellation, and terminal -// state handling consistent across start and SSH. +// state handling consistent across start and port forwarding. func waitForState(ctx context.Context, api *sandboxAPI, s *spinner, id, targetStatus, operation string, unexpectedStatuses ...string) (*sandboxEntry, error) { start := time.Now() deadline := start.Add(startWaitTimeout) @@ -133,3 +133,46 @@ func waitForState(ctx context.Context, api *sandboxAPI, s *spinner, id, targetSt } } } + +// ensureRunning starts the sandbox before port forwarding opens a connection. +// The caller must not have a spinner open. +func ensureRunning(ctx context.Context, api *sandboxAPI, id, currentStatus string) (*sandboxEntry, error) { + if strings.EqualFold(currentStatus, "running") { + return &sandboxEntry{SandboxID: id, Status: currentStatus}, nil + } + + s := spin(ctx, "Starting "+cmdio.Bold(ctx, id)+"…") + defer s.Close() + + var sb *sandboxEntry + if strings.EqualFold(currentStatus, "stopping") { + stopped, err := waitForStopped(ctx, api, s, id) + if err != nil { + s.fail("Failed to start " + id) + return nil, err + } + currentStatus = stopped.Status + s.Update("Starting " + id + "…") + } + + if strings.EqualFold(currentStatus, "stopped") { + updated, err := api.start(ctx, id) + if err != nil { + s.fail("Failed to start " + id) + return nil, fmt.Errorf("failed to start sandbox %s: %w", id, err) + } + sb = updated + } + + if sb == nil || !strings.EqualFold(sb.Status, "running") { + final, err := waitForRunning(ctx, api, s, id) + if err != nil { + s.fail("Failed to start " + id) + return nil, err + } + sb = final + } + + s.ok("Started " + cmdio.Bold(ctx, id)) + return sb, nil +} diff --git a/cmd/sandbox/start_test.go b/cmd/sandbox/start_test.go new file mode 100644 index 00000000000..1ab921f6042 --- /dev/null +++ b/cmd/sandbox/start_test.go @@ -0,0 +1,121 @@ +package sandbox + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnsureRunningLifecycle(t *testing.T) { + type requestStep struct { + method string + path string + status string + } + + for _, tc := range []struct { + name string + initialStatus string + steps []requestStep + wantStatus string + wantErr string + wantStarts int32 + }{ + { + name: "stopping waits for stopped before starting", + initialStatus: "Stopping", + steps: []requestStep{ + {method: http.MethodGet, path: sandboxPath("test-id"), status: "Stopped"}, + {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, + {method: http.MethodGet, path: sandboxPath("test-id"), status: "Running"}, + }, + wantStatus: "Running", + wantStarts: 1, + }, + { + name: "stopped starts normally", + initialStatus: "Stopped", + steps: []requestStep{ + {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, + {method: http.MethodGet, path: sandboxPath("test-id"), status: "Running"}, + }, + wantStatus: "Running", + wantStarts: 1, + }, + { + name: "running is a no-op", + initialStatus: "Running", + wantStatus: "Running", + }, + { + name: "stopped after start remains an error", + initialStatus: "Stopped", + steps: []requestStep{ + {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, + {method: http.MethodGet, path: sandboxPath("test-id"), status: "Stopped"}, + }, + wantErr: `sandbox test-id reached unexpected state "Stopped" while starting`, + wantStarts: 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var requestCount atomic.Int32 + var startCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + assert.NoError(t, json.NewEncoder(w).Encode(map[string]string{ + "oidc_endpoint": "https://workspace.example.test/oidc", + })) + return + } + + index := int(requestCount.Add(1) - 1) + if index >= len(tc.steps) { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + + step := tc.steps[index] + assert.Equal(t, step.method, r.Method) + assert.Equal(t, step.path, r.URL.Path) + if r.Method == http.MethodPost && r.URL.Path == sandboxPath("test-id")+"/start" { + startCount.Add(1) + } + assert.NoError(t, json.NewEncoder(w).Encode(sandboxEntry{ + SandboxID: "test-id", + Status: step.status, + })) + })) + t.Cleanup(server.Close) + + workspaceClient, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "test-token", + }) + require.NoError(t, err) + api, err := newSandboxAPI(workspaceClient) + require.NoError(t, err) + + ctx := cmdio.MockDiscard(t.Context()) + got, err := ensureRunning(ctx, api, "test-id", tc.initialStatus) + if tc.wantErr != "" { + require.EqualError(t, err, tc.wantErr) + assert.Nil(t, got) + } else { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, tc.wantStatus, got.Status) + } + assert.Equal(t, int32(len(tc.steps)), requestCount.Load()) + assert.Equal(t, tc.wantStarts, startCount.Load()) + }) + } +} diff --git a/experimental/ssh/README.md b/experimental/ssh/README.md index 67e8042fbef..917f5579c02 100644 --- a/experimental/ssh/README.md +++ b/experimental/ssh/README.md @@ -233,9 +233,12 @@ Resume protocol v2 lets an SSH session survive a dropped websocket without losin bytes. The protocol is symmetric: the client and server keep separate offsets and replay buffers for their outgoing streams. -Before opening the websocket, the client requests `/capabilities`. It enables resume only when the +For compute SSH, the client requests `/capabilities` before opening the websocket. It enables resume only when the server returns `{"resume_version":2}`. A missing route, another version, an invalid response, or a probe that takes more than ten seconds leaves resume disabled. The SSH connection still proceeds. +For sandbox SSH, the server reports `resume_version` in its command-output readiness response. +The client enables resume only for version 2 and authenticates each replacement WebSocket through +sandbox port forwarding using current workspace credentials and the invocation token. When resume is enabled, the initial websocket URL includes these query parameters: | Parameter | Initial value | Meaning | diff --git a/experimental/ssh/internal/client/releases.go b/experimental/ssh/internal/client/releases.go index def34ce363c..a38d6bff29b 100644 --- a/experimental/ssh/internal/client/releases.go +++ b/experimental/ssh/internal/client/releases.go @@ -9,12 +9,12 @@ import ( "io/fs" "net" "net/http" - "os" "path/filepath" "strings" "syscall" "time" + "github.com/databricks/cli/experimental/ssh/internal/releases" "github.com/databricks/cli/experimental/ssh/internal/workspace" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/filer" @@ -176,39 +176,12 @@ func isProxyUploadError(err error) bool { return strings.Contains(msg, "stream error") && strings.Contains(msg, "stream ID") } -func getReleaseName(architecture, version string) string { - if strings.Contains(version, "dev") { - return fmt.Sprintf("databricks_cli_linux_%s.zip", architecture) - } - return fmt.Sprintf("databricks_cli_%s_linux_%s.zip", version, architecture) -} +func getReleaseName(architecture, version string) string { return releases.Name(architecture, version) } func getLocalRelease(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { - log.Infof(ctx, "Looking for CLI releases in directory: %s", releasesDir) - releaseName := getReleaseName(architecture, version) - releasePath := filepath.Join(releasesDir, releaseName) - file, err := os.Open(releasePath) - if err != nil { - return nil, fmt.Errorf("failed to open %s: %w", releasePath, err) - } - return file, nil + return releases.OpenLocal(ctx, architecture, version, releasesDir) } func getGithubRelease(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { - // TODO: download and check databricks_cli__SHA256SUMS - fileName := getReleaseName(architecture, version) - downloadURL := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, fileName) - log.Infof(ctx, "Downloading %s from %s", fileName, downloadURL) - - resp, err := http.Get(downloadURL) - if err != nil { - return nil, fmt.Errorf("failed to download %s: %w", downloadURL, err) - } - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("failed to download %s: HTTP %d", downloadURL, resp.StatusCode) - } - - return resp.Body, nil + return releases.OpenPublished(ctx, architecture, version, releasesDir) } diff --git a/experimental/ssh/internal/releases/releases.go b/experimental/ssh/internal/releases/releases.go new file mode 100644 index 00000000000..98155730e6c --- /dev/null +++ b/experimental/ssh/internal/releases/releases.go @@ -0,0 +1,59 @@ +package releases + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/log" +) + +// Name returns the Linux CLI release archive name for the architecture and version. +func Name(architecture, version string) string { + if strings.Contains(version, "dev") { + return fmt.Sprintf("databricks_cli_linux_%s.zip", architecture) + } + return fmt.Sprintf("databricks_cli_%s_linux_%s.zip", version, architecture) +} + +// OpenLocal opens the matching Linux CLI release archive from releasesDir. +// The caller must close the returned reader. +func OpenLocal(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + log.Infof(ctx, "Looking for CLI releases in directory: %s", releasesDir) + releaseName := Name(architecture, version) + releasePath := filepath.Join(releasesDir, releaseName) + file, err := os.Open(releasePath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %w", releasePath, err) + } + return file, nil +} + +// OpenPublished downloads the matching Linux CLI release archive. +// The caller must close the returned reader. +func OpenPublished(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + // TODO: download and check databricks_cli__SHA256SUMS + fileName := Name(architecture, version) + downloadURL := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, fileName) + log.Infof(ctx, "Downloading %s from %s", fileName, downloadURL) + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(request) + if err != nil { + return nil, fmt.Errorf("failed to download %s: %w", downloadURL, err) + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("failed to download %s: HTTP %d", downloadURL, resp.StatusCode) + } + + return resp.Body, nil +} diff --git a/experimental/ssh/internal/releases/releases_test.go b/experimental/ssh/internal/releases/releases_test.go new file mode 100644 index 00000000000..b5c986762d1 --- /dev/null +++ b/experimental/ssh/internal/releases/releases_test.go @@ -0,0 +1,41 @@ +package releases_test + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/releases" + "github.com/stretchr/testify/require" +) + +type stalledDownload struct{ started chan struct{} } + +func (download stalledDownload) RoundTrip(request *http.Request) (*http.Response, error) { + close(download.started) + <-request.Context().Done() + return nil, request.Context().Err() +} + +func TestPublishedDownloadHonorsCancellation(t *testing.T) { + started := make(chan struct{}) + previous := http.DefaultTransport + http.DefaultTransport = stalledDownload{started: started} + t.Cleanup(func() { http.DefaultTransport = previous }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + finished := make(chan error, 1) + go func() { + _, err := releases.OpenPublished(ctx, "amd64", "1.0.0", "") + finished <- err + }() + <-started + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("release download ignored cancellation") + } +} diff --git a/experimental/ssh/sandbox/cache.py b/experimental/ssh/sandbox/cache.py new file mode 100644 index 00000000000..7fa5b9ae01f --- /dev/null +++ b/experimental/ssh/sandbox/cache.py @@ -0,0 +1,66 @@ +import fcntl +import hashlib +import json +import os +import re +import shutil +import sys +import time +from pathlib import Path + +MAX_CACHED_RELEASES = 8 +MAX_CACHE_AGE = 30 * 86400 + +operation, target, session = sys.argv[1:4] +target = Path(target) +session = Path(session) +releases = target.parent.parent +releases.mkdir(mode=0o700, parents=True, exist_ok=True) +# Keep this lock outside the pruned directories so concurrent starts share it. +with open(releases / "install.lock", "a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + cached = target.is_file() and os.access(target, os.X_OK) + if operation == "install": + temporary = Path(sys.argv[4]) + try: + with temporary.open("rb") as source: + digest = ( + hashlib.file_digest(source, "sha256").hexdigest() + if hasattr(hashlib, "file_digest") + else hashlib.sha256(source.read()).hexdigest() + ) + if digest != sys.argv[5]: + raise ValueError("Uploaded CLI executable checksum mismatch") + if not cached: + target.parent.mkdir(mode=0o700, exist_ok=True) + temporary.chmod(0o700) + os.replace(temporary, target) + cached = True + finally: + if temporary.exists(): + temporary.unlink() + elif operation != "check": + raise ValueError("Unknown executable cache operation") + if cached: + # Pin before releasing the lock, including the gap before server exec. + # Session cleanup already preserves directories with a live server lease. + lease = session / target.parent.name + if not lease.exists(): + os.link(target, lease) + os.utime(target.parent, None) + entries = [ + entry + for entry in releases.iterdir() + if re.fullmatch(r"(?:amd64|arm64)-[0-9a-f]{64}", entry.name) and not entry.is_symlink() and entry.is_dir() + ] + # Leased releases may exceed the budget until their sessions are cleaned up. + retained = int(target.parent.exists()) + for entry in sorted(entries, key=lambda entry: entry.stat().st_mtime, reverse=True): + executable = entry / "databricks" + if entry == target.parent or (executable.exists() and executable.stat().st_nlink > 1): + continue + if entry.stat().st_mtime >= time.time() - MAX_CACHE_AGE and retained < MAX_CACHED_RELEASES: + retained += 1 + continue + shutil.rmtree(entry) + print(json.dumps({"cached": cached})) diff --git a/experimental/ssh/sandbox/command.go b/experimental/ssh/sandbox/command.go index f25c0c67d11..323ed76282a 100644 --- a/experimental/ssh/sandbox/command.go +++ b/experimental/ssh/sandbox/command.go @@ -1,8 +1,10 @@ package sandbox import ( + "bytes" "errors" "fmt" + "io" "net" "os" "os/signal" @@ -37,9 +39,15 @@ func NewServerCommand() *cobra.Command { } ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGTERM, os.Interrupt) defer stop() + output, err := openServerOutput(cmd.OutOrStdout()) + if err != nil { + listener.Close() + return fmt.Errorf("open command output: %w", err) + } + defer output.Close() return Serve(ctx, listener, ServerOptions{ - SSHD: sshd, Config: config, ConnectTimeout: connectTimeout, - }, cmd.OutOrStdout()) + SSHD: sshd, Config: config, ConnectTimeout: connectTimeout, OutputHeartbeat: 30 * time.Second, + }, output) }, } cmd.Flags().StringVar(&config, "sshd-config", "", "Prepared sshd configuration file") @@ -53,3 +61,25 @@ func NewServerCommand() *cobra.Command { func validForwardPort(port int) bool { return port >= 10000 && port <= 65535 && port != 10101 && port != 18767 && port != 19090 } + +func openServerOutput(output io.Writer) (io.WriteCloser, error) { + if _, ok := output.(*bytes.Buffer); ok || output == io.Discard { + return serverOutputWriter{output}, nil + } + file, ok := output.(*os.File) + if !ok { + return nil, errors.New("server output must be a Linux command-execution pipe") + } + info, err := file.Stat() + if err != nil { + return nil, err + } + if info.Mode()&os.ModeNamedPipe == 0 { + return nil, errors.New("server output must be a Linux command-execution pipe") + } + return duplicateServerOutput(file) +} + +type serverOutputWriter struct{ io.Writer } + +func (serverOutputWriter) Close() error { return nil } diff --git a/experimental/ssh/sandbox/command_ssh.go b/experimental/ssh/sandbox/command_ssh.go new file mode 100644 index 00000000000..f5273c9def9 --- /dev/null +++ b/experimental/ssh/sandbox/command_ssh.go @@ -0,0 +1,256 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "runtime" + "strings" + "syscall" + "unicode" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/execv" + "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/spf13/cobra" +) + +// CommandDependencies supplies sandbox selection and completion from the stable commands. +type CommandDependencies struct { + ResolveName func(context.Context, *databricks.WorkspaceClient, string) (string, error) + Complete func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) +} + +// NewSSHCommand connects to a sandbox through authenticated WebSocket forwarding. +func NewSSHCommand(deps CommandDependencies) *cobra.Command { + var proxy bool + var releasesDir string + var expectedIdentity string + var configFile string + cmd := &cobra.Command{ + Use: "ssh [sandbox-id] [-- ...]", + Short: "SSH into a Sandbox environment", + Long: `Connect to your default or a named sandbox through authenticated WebSocket port forwarding. + +The CLI creates a local SSH key automatically. No Gateway SSH key registration +or persistent SSH Host configuration is required. The sandbox must have Python 3 +and openssh-server installed, and the local machine must have OpenSSH. + +Examples: + databricks sandbox ssh + databricks sandbox ssh happy-panda-1234 + databricks sandbox ssh happy-panda-1234 -- cat /etc/os-release + databricks sandbox ssh -- -L 8080:localhost:8080`, + Args: cobra.ArbitraryArgs, ValidArgsFunction: deps.Complete, + PreRunE: func(cmd *cobra.Command, args []string) error { + if configFile == "" { + return root.MustWorkspaceClient(cmd, args) + } + if !proxy { + return errors.New("--ssh-config-file requires --proxy") + } + // The SDK reads this setting from process environment during initialization. + previous, present := env.Lookup(cmd.Context(), "DATABRICKS_CONFIG_FILE") + if err := os.Setenv("DATABRICKS_CONFIG_FILE", configFile); err != nil { + return err + } + defer func() { + if present { + _ = os.Setenv("DATABRICKS_CONFIG_FILE", previous) + } else { + _ = os.Unsetenv("DATABRICKS_CONFIG_FILE") + } + }() + return root.MustWorkspaceClient(cmd, args) + }, + RunE: func(cmd *cobra.Command, args []string) error { + var requested string + var extra []string + dash := cmd.ArgsLenAtDash() + if dash == -1 { + dash = len(args) + } + if dash > 1 { + return errors.New("specify at most one sandbox before --") + } + if dash == 1 { + requested = args[0] + } + extra = args[dash:] + if proxy && len(extra) != 0 { + return errors.New("--proxy does not accept SSH arguments") + } + ctx := cmd.Context() + if proxy { + var stop context.CancelFunc + ctx, stop = signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) + defer stop() + } + w := cmdctx.WorkspaceClient(ctx) + name, err := deps.ResolveName(ctx, w, requested) + if err != nil { + return err + } + identity, err := prepareSSHIdentity(ctx, w, name) + if err != nil { + return err + } + if expectedIdentity != "" && identity.HostKeyAlias != expectedIdentity { + return errors.New("SSH workspace or caller changed; rerun ssh-setup with the intended Databricks profile") + } + if proxy { + return ConnectProxy(ctx, w, name, identity, releasesDir, os.Stdin, cmd.OutOrStdout(), cmd.ErrOrStderr()) + } + options, err := sshClientOptions(cmd, w.Config, name, identity, releasesDir, extra) + if err != nil { + return err + } + return execv.Execv(options) + }, + } + cmd.Flags().BoolVar(&proxy, "proxy", false, "Relay SSH bytes over stdin/stdout without launching ssh") + cmd.Flags().MarkHidden("proxy") + cmd.Flags().StringVar(&expectedIdentity, "ssh-identity", "", "Expected local SSH identity scope") + cmd.Flags().MarkHidden("ssh-identity") + cmd.Flags().StringVar(&configFile, "ssh-config-file", "", "Databricks configuration file saved by ssh-setup") + cmd.Flags().MarkHidden("ssh-config-file") + cmd.Flags().StringVar(&releasesDir, "releases-dir", "", "Directory containing version-matched Linux CLI archives") + return cmd +} + +func sshClientOptions(cmd *cobra.Command, cfg *config.Config, name string, identity *Identity, releasesDir string, extra []string) (execv.Options, error) { + executable, err := os.Executable() + if err != nil { + return execv.Options{}, err + } + profile := "" + if flag := cmd.Flag("profile"); flag != nil && flag.Changed { + profile = flag.Value.String() + } + target := "" + for _, flagName := range []string{"target", "environment"} { + if flag := cmd.Flag(flagName); flag != nil && flag.Value.String() != "" { + target = flag.Value.String() + break + } + } + transport, err := sshProxyCommand(executable, name, sshProxyOptions{Profile: profile, ReleasesDir: releasesDir, IdentityScope: identity.HostKeyAlias, Target: target}) + if err != nil { + return execv.Options{}, err + } + return execv.Options{Args: buildSSHArgs(identity, transport, extra), Env: sshProcessEnvironment(cmd.Context(), cfg)}, nil +} + +func sshProcessEnvironment(ctx context.Context, cfg *config.Config) []string { + variables := env.All(ctx) + if variables["DATABRICKS_HOST"] != cfg.Host { + delete(variables, "DATABRICKS_ACCOUNT_ID") + delete(variables, "DATABRICKS_WORKSPACE_ID") + } + variables["DATABRICKS_HOST"] = cfg.Host + if cfg.AccountID != "" { + variables["DATABRICKS_ACCOUNT_ID"] = cfg.AccountID + } + if cfg.WorkspaceID != "" { + variables["DATABRICKS_WORKSPACE_ID"] = cfg.WorkspaceID + } + if cfg.Profile != "" { + // An implicit profile must retain environment-first auth precedence. + // Converting it to --profile would give the profile precedence instead. + variables["DATABRICKS_CONFIG_PROFILE"] = cfg.Profile + } + if cfg.ConfigFile != "" { + variables["DATABRICKS_CONFIG_FILE"] = cfg.ConfigFile + } + result := make([]string, 0, len(variables)) + for key, value := range variables { + result = append(result, key+"="+value) + } + return result +} + +func prepareSSHIdentity(ctx context.Context, w *databricks.WorkspaceClient, name string) (*Identity, error) { + caller, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) + if err != nil { + return nil, fmt.Errorf("resolve SSH caller: %w", err) + } + workspace, err := auth.ResolveWorkspaceID(ctx, w) + if err != nil { + return nil, fmt.Errorf("resolve SSH workspace: %w", err) + } + return LocalIdentity(ctx, w.Config.Host+"/"+workspace, caller.Id, name) +} + +type sshProxyOptions struct { + Profile, ReleasesDir, IdentityScope, Target, ConfigFile string +} + +func sshProxyCommand(executable, name string, options sshProxyOptions) (string, error) { + args := []string{executable, "sandbox", "ssh", name, "--proxy"} + if options.Profile != "" { + args = append(args, "--profile", options.Profile) + } + if options.ReleasesDir != "" { + args = append(args, "--releases-dir", options.ReleasesDir) + } + if options.IdentityScope != "" { + args = append(args, "--ssh-identity", options.IdentityScope) + } + if options.Target != "" { + args = append(args, "--target", options.Target) + } + if options.ConfigFile != "" { + args = append(args, "--ssh-config-file", options.ConfigFile) + } + for i, arg := range args { + // OpenSSH expands percent tokens before invoking the shell, even inside quotes. + if strings.ContainsFunc(arg, unicode.IsControl) { + return "", errors.New("control characters are not supported in SSH proxy arguments") + } + arg = strings.ReplaceAll(arg, "%", "%%") + if runtime.GOOS == "windows" { + quoted, ok := shellquote.WindowsArg(arg) + if !ok { + return "", fmt.Errorf("unsupported Windows SSH proxy argument %q", arg) + } + args[i] = quoted + } else { + args[i] = shellquote.BashArg(arg) + } + } + return strings.Join(args, " "), nil +} + +func buildSSHArgs(identity *Identity, proxyCommand string, extraArgs []string) []string { + args := []string{ + "ssh", "-i", identity.KeyPath, "-o", "IdentitiesOnly=yes", "-o", "PreferredAuthentications=publickey", + // Include authentication and the proxy's ten-minute remote setup budget. + "-o", "ConnectTimeout=900", + // Each invocation owns its server; do not reuse a user's multiplexed connection. + "-o", "ControlPath=none", + "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + quoteSSHOption(identity.KnownHostsPath), + "-o", "HostKeyAlias=" + identity.HostKeyAlias, "-o", "ServerAliveInterval=30", "-o", "ProxyCommand=" + proxyCommand, + "sandbox-agent@" + identity.HostKeyAlias, + } + if len(extraArgs) == 1 { + return append(args, extraArgs[0]) + } + for _, arg := range extraArgs { + args = append(args, shellquote.BashArg(arg)) + } + return args +} + +func quoteSSHOption(value string) string { + return `"` + strings.ReplaceAll(filepathToSSH(value), `"`, `\"`) + `"` +} + +func filepathToSSH(value string) string { return strings.ReplaceAll(value, `\`, "/") } diff --git a/experimental/ssh/sandbox/command_ssh_test.go b/experimental/ssh/sandbox/command_ssh_test.go new file mode 100644 index 00000000000..b41520a56cc --- /dev/null +++ b/experimental/ssh/sandbox/command_ssh_test.go @@ -0,0 +1,374 @@ +package sandbox + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + sandboxcmd "github.com/databricks/cli/cmd/sandbox" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildSSHArgsBaseFlags(t *testing.T) { + identity := &Identity{KeyPath: "/keys/id", KnownHostsPath: "/keys/known hosts", HostKeyAlias: "sandbox-host"} + got := buildSSHArgs(identity, "databricks sandbox ssh id --proxy", nil) + assert.Equal(t, "sandbox-agent@sandbox-host", got[len(got)-1]) + assert.Contains(t, got, "StrictHostKeyChecking=yes") + assert.Contains(t, got, "ControlPath=none") + assert.Contains(t, got, `UserKnownHostsFile="/keys/known hosts"`) + assert.Contains(t, got, "ProxyCommand=databricks sandbox ssh id --proxy") +} + +func TestBuildSSHArgsIgnoresInheritedControlPath(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires the local Unix OpenSSH client") + } + if _, err := exec.LookPath("ssh"); err != nil { + t.Skip("requires OpenSSH") + } + t.Setenv("SHELL", "/bin/sh") + configPath := filepath.Join(t.TempDir(), "config") + config := "Host *\n ControlMaster auto\n ControlPath " + filepath.Join(t.TempDir(), "%r@%h:%p") + "\n" + require.NoError(t, os.WriteFile(configPath, []byte(config), 0o600)) + identity := &Identity{KeyPath: "/nonexistent/key", KnownHostsPath: "/nonexistent/known_hosts", HostKeyAlias: "sandbox-" + strings.Repeat("a", 64)} + marker := filepath.Join(t.TempDir(), "proxy-started") + args := buildSSHArgs(identity, "touch "+shellquote.BashArg(marker), nil) + command := exec.CommandContext(t.Context(), "ssh", append([]string{"-F", configPath}, args[1:]...)...) + output, err := command.CombinedOutput() + require.Error(t, err, "the test proxy exits before SSH negotiation") + assert.FileExists(t, marker, "OpenSSH must reach its ProxyCommand despite the long configured ControlPath") + assert.NotContains(t, string(output), "ControlPath too long") +} + +func TestSSHClientOptionsPreserveProfileAndTarget(t *testing.T) { + for _, test := range []struct { + name string + flags []string + profile string + target string + }{ + {name: "explicit profile and target", flags: []string{"--profile", "work", "--target", "dev"}, profile: "work", target: "dev"}, + {name: "implicit profile"}, + {name: "deprecated environment flag", flags: []string{"--environment", "dev"}, target: "dev"}, + {name: "target takes precedence", flags: []string{"--target", "dev", "--environment", "prod"}, target: "dev"}, + } { + t.Run(test.name, func(t *testing.T) { + cmd := newSSHCommand() + cmd.SetContext(env.Set(t.Context(), "DATABRICKS_TOKEN", "environment-token")) + cmd.Flags().String("profile", "", "Databricks profile") + cmd.Flags().String("target", "", "Bundle target") + cmd.Flags().String("environment", "", "Deprecated bundle target") + require.NoError(t, cmd.ParseFlags(test.flags)) + cfg := &config.Config{Host: "https://workspace.test", WorkspaceID: "123", Profile: "work"} + identity := &Identity{KeyPath: "/keys/id", KnownHostsPath: "/keys/known-hosts", HostKeyAlias: "sandbox-scope"} + options, err := sshClientOptions(cmd, cfg, "sandboxes/test", identity, "/releases", []string{"uname", "-a"}) + require.NoError(t, err) + var proxyCommand string + for _, arg := range options.Args { + if strings.HasPrefix(arg, "ProxyCommand=") { + proxyCommand = arg + } + } + assert.Contains(t, proxyCommand, "sandbox ssh sandboxes/test --proxy") + assert.Contains(t, proxyCommand, "--ssh-identity sandbox-scope") + assert.Contains(t, proxyCommand, "--releases-dir /releases") + if test.profile == "" { + assert.NotContains(t, proxyCommand, "--profile") + } else { + assert.Contains(t, proxyCommand, "--profile "+test.profile) + } + if test.target == "" { + assert.NotContains(t, proxyCommand, "--target") + } else { + assert.Contains(t, proxyCommand, "--target "+test.target) + } + assert.Equal(t, []string{"sandbox-agent@sandbox-scope", "uname", "-a"}, options.Args[len(options.Args)-3:]) + assert.Contains(t, options.Env, "DATABRICKS_HOST=https://workspace.test") + assert.Contains(t, options.Env, "DATABRICKS_WORKSPACE_ID=123") + assert.Contains(t, options.Env, "DATABRICKS_CONFIG_PROFILE=work") + assert.Contains(t, options.Env, "DATABRICKS_TOKEN=environment-token") + assert.Contains(t, options.Args, "ConnectTimeout=900") + }) + } +} + +func TestSSHProcessEnvironmentPreservesResolvedWorkspaceAndCredentials(t *testing.T) { + ctx := env.Set(t.Context(), "DATABRICKS_HOST", "https://old-workspace.test") + ctx = env.Set(ctx, "DATABRICKS_TOKEN", "environment-token") + ctx = env.Set(ctx, "DATABRICKS_WORKSPACE_ID", "old-workspace") + configuration := &config.Config{Host: "https://selected-workspace.test", WorkspaceID: "123", AccountID: "selected-account", Profile: "work", ConfigFile: "/profiles/databrickscfg"} + actual := map[string]string{} + for _, variable := range sshProcessEnvironment(ctx, configuration) { + key, value, _ := strings.Cut(variable, "=") + actual[key] = value + } + assert.Equal(t, configuration.Host, actual["DATABRICKS_HOST"]) + assert.Equal(t, "123", actual["DATABRICKS_WORKSPACE_ID"]) + assert.Equal(t, "selected-account", actual["DATABRICKS_ACCOUNT_ID"]) + assert.Equal(t, "work", actual["DATABRICKS_CONFIG_PROFILE"]) + assert.Equal(t, configuration.ConfigFile, actual["DATABRICKS_CONFIG_FILE"]) + assert.Equal(t, "environment-token", actual["DATABRICKS_TOKEN"]) +} + +func TestSSHProcessEnvironmentPreservesInheritedIDsWhenUnresolved(t *testing.T) { + for _, test := range []struct { + name, parentHost, accountID, workspaceID, wantAccount, wantWorkspace string + }{ + {"same host unresolved", "https://workspace.test", "", "", "inherited-account", "456"}, + {"same host resolved account", "https://workspace.test", "selected-account", "", "selected-account", "456"}, + {"same host resolved workspace", "https://workspace.test", "", "123", "inherited-account", "123"}, + {"changed host unresolved", "https://old-workspace.test", "", "", "", ""}, + {"changed host resolved account", "https://old-workspace.test", "selected-account", "", "selected-account", ""}, + {"changed host resolved workspace", "https://old-workspace.test", "", "123", "", "123"}, + {"missing host unresolved", "", "", "", "", ""}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := env.Set(t.Context(), "DATABRICKS_ACCOUNT_ID", "inherited-account") + ctx = env.Set(ctx, "DATABRICKS_WORKSPACE_ID", "456") + ctx = env.Set(ctx, "DATABRICKS_HOST", test.parentHost) + cfg := &config.Config{Host: "https://workspace.test", AccountID: test.accountID, WorkspaceID: test.workspaceID} + actual := map[string]string{} + for _, variable := range sshProcessEnvironment(ctx, cfg) { + key, value, _ := strings.Cut(variable, "=") + actual[key] = value + } + assert.Equal(t, test.wantAccount, actual["DATABRICKS_ACCOUNT_ID"]) + assert.Equal(t, test.wantWorkspace, actual["DATABRICKS_WORKSPACE_ID"]) + }) + } +} + +func TestSSHProxyLoadsSavedConfigFile(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + filename := filepath.Join(t.TempDir(), "selected config") + require.NoError(t, os.WriteFile(filename, []byte("[work]\nhost = "+server.URL+"\ntoken = saved-token\n"), 0o600)) + t.Setenv("DATABRICKS_CONFIG_FILE", filepath.Join(t.TempDir(), "missing")) + previous := os.Getenv("DATABRICKS_CONFIG_FILE") + cmd := newSSHCommand() + cmd.SetContext(t.Context()) + cmd.Flags().String("profile", "", "Databricks profile") + require.NoError(t, cmd.ParseFlags([]string{"--proxy", "--profile", "work", "--ssh-config-file", filename})) + require.NoError(t, cmd.PreRunE(cmd, nil)) + cfg := cmdctx.WorkspaceClient(cmd.Context()).Config + assert.Equal(t, filename, cfg.ConfigFile) + assert.Equal(t, server.URL, cfg.Host) + assert.Equal(t, "saved-token", cfg.Token) + assert.Equal(t, previous, os.Getenv("DATABRICKS_CONFIG_FILE")) + + cmd = newSSHCommand() + require.NoError(t, cmd.ParseFlags([]string{"--ssh-config-file", filename})) + require.ErrorContains(t, cmd.PreRunE(cmd, nil), "requires --proxy") +} + +func TestBuildSSHArgsQuoting(t *testing.T) { + for _, tc := range []struct { + name string + extraArgs []string + // expected is what we expect at the end of args, after the destination. + expected []string + }{ + { + name: "no extras", + extraArgs: nil, + expected: nil, + }, + { + name: "single safe word — passed through", + extraArgs: []string{"uname"}, + expected: []string{"uname"}, + }, + { + name: "single string with spaces — passed through so the remote shell can split it", + extraArgs: []string{"echo hello-from-string"}, + expected: []string{"echo hello-from-string"}, + }, + { + name: "single string with a pipe — passed through to the remote shell", + extraArgs: []string{"cat /etc/os-release | head -3"}, + expected: []string{"cat /etc/os-release | head -3"}, + }, + { + name: "multi safe words — no quoting", + extraArgs: []string{"ls", "-la", "/tmp"}, + expected: []string{"ls", "-la", "/tmp"}, + }, + { + name: "bash -c '' — third arg gets quoted", + // The whole point: without the quoting, the remote shell + // re-splits "bash -c echo hi" and bash's -c eats just "echo". + extraArgs: []string{"bash", "-c", "echo hi"}, + expected: []string{"bash", "-c", "'echo hi'"}, + }, + { + name: "arg with single quote — escaped, not lost", + extraArgs: []string{"echo", "it's fine"}, + expected: []string{"echo", `'it'\''s fine'`}, + }, + { + name: "glob pattern — quoted so the remote (not the local) shell expands it", + extraArgs: []string{"find", ".", "-name", "*.txt"}, + expected: []string{"find", ".", "-name", "'*.txt'"}, + }, + { + name: "ssh -o key=val style — gets quoted because of '='; harmless given current arg ordering", + extraArgs: []string{"-o", "StrictHostKeyChecking=no"}, + expected: []string{"-o", "'StrictHostKeyChecking=no'"}, + }, + { + name: "empty arg gets explicit empty quotes (not lost)", + extraArgs: []string{"sh", "-c", ""}, + expected: []string{"sh", "-c", "''"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := buildSSHArgs(&Identity{HostKeyAlias: "host"}, "proxy", tc.extraArgs) + // Trim everything up to and including the destination so we + // can assert just on the appended extras. + dest := "sandbox-agent@host" + cut := -1 + for i, a := range got { + if a == dest { + cut = i + 1 + break + } + } + assert.GreaterOrEqual(t, cut, 0, "destination not found in args") + tail := got[cut:] + if tc.expected == nil { + assert.Empty(t, tail) + } else { + assert.Equal(t, tc.expected, tail) + } + }) + } +} + +func TestSSHProxyCommandQuotesArguments(t *testing.T) { + command, err := sshProxyCommand("/path with spaces/databricks", "sandboxes/test", sshProxyOptions{Profile: "work profile", ReleasesDir: "/release dir"}) + require.NoError(t, err) + if runtime.GOOS == "windows" { + assert.Equal(t, `"/path with spaces/databricks" sandbox ssh sandboxes/test --proxy --profile "work profile" --releases-dir "/release dir"`, command) + } else { + assert.Equal(t, "'/path with spaces/databricks' sandbox ssh sandboxes/test --proxy --profile 'work profile' --releases-dir '/release dir'", command) + } + percent, percentErr := sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "a%h"}) + if runtime.GOOS == "windows" { + require.ErrorContains(t, percentErr, "unsupported Windows SSH proxy argument") + } else { + require.NoError(t, percentErr) + assert.Equal(t, "databricks sandbox ssh sandboxes/test --proxy --profile 'a%%h'", percent) + } + _, err = sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "bad\nprofile"}) + require.ErrorContains(t, err, "control characters") + scoped, err := sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "work", IdentityScope: "sandbox-scope", Target: "dev"}) + require.NoError(t, err) + assert.Equal(t, "databricks sandbox ssh sandboxes/test --proxy --profile work --ssh-identity sandbox-scope --target dev", scoped) + saved, err := sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "work", ConfigFile: "/profiles/custom config"}) + require.NoError(t, err) + if runtime.GOOS == "windows" { + assert.Equal(t, `databricks sandbox ssh sandboxes/test --proxy --profile work --ssh-config-file "/profiles/custom config"`, saved) + } else { + assert.Equal(t, "databricks sandbox ssh sandboxes/test --proxy --profile work --ssh-config-file '/profiles/custom config'", saved) + } +} + +func TestSSHProxyCommandDispatchAndIdentityMismatch(t *testing.T) { + for _, mismatch := range []bool{false, true} { + t.Run(strconv.FormatBool(mismatch), func(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + var paths []string + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + switch r.URL.Path { + case "/.well-known/databricks-config": + _, _ = io.WriteString(w, `{}`) + case "/api/2.0/preview/scim/v2/Me": + _, _ = io.WriteString(w, `{"id":"caller"}`) + default: + _, _ = io.WriteString(w, `{"status":{"state":"SANDBOX_STATE_UNSPECIFIED"}}`) + } + })) + defer endpoint.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", WorkspaceID: "123"}) + require.NoError(t, err) + cmd := newSSHCommand() + cmd.PreRunE = nil + args := []string{"test", "--proxy"} + if mismatch { + args = append(args, "--ssh-identity", "different-workspace") + } + cmd.SetArgs(args) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + err = cmd.ExecuteContext(cmdctx.SetWorkspaceClient(ctx, w)) + if mismatch { + require.ErrorContains(t, err, "workspace or caller changed") + } else { + require.ErrorContains(t, err, "unsupported state") + } + assert.Contains(t, paths, "/api/2.0/sandboxes/test") + assert.Contains(t, paths, "/api/2.0/preview/scim/v2/Me") + }) + } +} + +func TestSSHCommandRejectsExtraSandboxArguments(t *testing.T) { + cmd := newSSHCommand() + // Argument validation runs before credentials are resolved by RunE. + cmd.PreRunE = nil + cmd.SetArgs([]string{"first", "second"}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "at most one sandbox") + cmd = newSSHCommand() + cmd.PreRunE = nil + cmd.SetArgs([]string{"first", "--proxy", "--", "echo"}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "--proxy does not accept SSH arguments") +} + +func TestPrepareSSHIdentityUsesWorkspaceAndCaller(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + failure := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/preview/scim/v2/Me", r.URL.Path) + if failure { + http.Error(w, `{"error_code":"PERMISSION_DENIED","message":"denied"}`, http.StatusForbidden) + return + } + _, _ = io.WriteString(w, `{"id":"caller"}`) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token", WorkspaceID: "123"}) + require.NoError(t, err) + got, err := prepareSSHIdentity(ctx, w, "sandboxes/test") + require.NoError(t, err) + expected, err := LocalIdentity(ctx, server.URL+"/123", "caller", "sandboxes/test") + require.NoError(t, err) + assert.Equal(t, expected.KeyPath, got.KeyPath) + failure = true + _, err = prepareSSHIdentity(ctx, w, "sandboxes/test") + require.ErrorContains(t, err, "resolve SSH caller") +} + +func newSSHCommand() *cobra.Command { + return NewSSHCommand(CommandDependencies{ResolveName: sandboxcmd.ResolveSSHName, Complete: sandboxcmd.CompleteSandboxIDs}) +} diff --git a/experimental/ssh/sandbox/command_test.go b/experimental/ssh/sandbox/command_test.go index 2690d8953ee..dccd7c42c70 100644 --- a/experimental/ssh/sandbox/command_test.go +++ b/experimental/ssh/sandbox/command_test.go @@ -1,28 +1,88 @@ -package sandbox_test +package sandbox import ( + "bufio" "bytes" + "context" "encoding/json" + "fmt" + "io" + "io/fs" "net" "os" + "os/exec" "path/filepath" "runtime" "strconv" "testing" - - "github.com/databricks/cli/experimental/ssh/sandbox" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestServerOutputInterruptsInheritedStdout(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sandbox command output uses Linux pipes") + } + const marker = "server output write interrupted\n" + if mode := os.Getenv("DATABRICKS_TEST_SERVER_OUTPUT"); mode != "" { + // An inherited pipe remains writable even when its permissions prevent reopening. + require.NoError(t, os.Stdout.Chmod(0)) + output, err := openServerOutput(os.Stdout) + require.NoError(t, err) + defer output.Close() + file := output.(*os.File) + if mode == "deadline" { + require.NoError(t, file.SetWriteDeadline(time.Now().Add(100*time.Millisecond))) + } else { + timer := time.AfterFunc(100*time.Millisecond, func() { file.Close() }) + defer timer.Stop() + } + _, err = file.Write(make([]byte, 16<<20)) + if mode == "deadline" { + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + } else { + require.ErrorIs(t, err, fs.ErrClosed) + } + fmt.Fprint(os.Stderr, marker) + return + } + for _, mode := range []string{"deadline", "close"} { + t.Run(mode, func(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, executable, "-test.run=^TestServerOutputInterruptsInheritedStdout$") + cmd.Env = append(os.Environ(), "DATABRICKS_TEST_SERVER_OUTPUT="+mode) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + stderr, err := cmd.StderrPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + // Keep stdout full until the child's write has been interrupted. + line, err := bufio.NewReader(stderr).ReadString('\n') + require.NoError(t, err) + require.Equal(t, marker, line) + _, err = io.Copy(io.Discard, stdout) + require.NoError(t, err) + require.NoError(t, cmd.Wait()) + }) + } +} + func TestServerCommandValidatesArguments(t *testing.T) { for _, args := range [][]string{ {"--sshd-config", "config", "--port", "22"}, {"--sshd-config", "config", "--connect-timeout", "0s"}, {"--sshd-config", "config", "unexpected"}, } { - cmd := sandbox.NewServerCommand() + cmd := NewServerCommand() cmd.SetArgs(args) err := cmd.ExecuteContext(t.Context()) require.Error(t, err) @@ -35,7 +95,7 @@ func TestServerCommandReportsReadiness(t *testing.T) { } binary := filepath.Join(t.TempDir(), "sshd") require.NoError(t, os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o700)) - cmd := sandbox.NewServerCommand() + cmd := NewServerCommand() var output bytes.Buffer cmd.SetOut(&output) cmd.SetArgs([]string{"--sshd", binary, "--sshd-config", "prepared-config", "--port", "0", "--connect-timeout", "20ms"}) @@ -51,7 +111,7 @@ func TestServerCommandReportsOccupiedPort(t *testing.T) { listener, err := net.Listen("tcp4", "0.0.0.0:0") require.NoError(t, err) defer listener.Close() - cmd := sandbox.NewServerCommand() + cmd := NewServerCommand() cmd.SetArgs([]string{"--sshd-config", "prepared-config", "--port", strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)}) require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "listen for SSH WebSockets") } @@ -59,7 +119,7 @@ func TestServerCommandReportsOccupiedPort(t *testing.T) { func TestServerCommandRejectsNonForwardablePorts(t *testing.T) { for _, port := range []string{"9999", "10101", "18767", "19090", "65536"} { t.Run(port, func(t *testing.T) { - cmd := sandbox.NewServerCommand() + cmd := NewServerCommand() cmd.SetArgs([]string{"--sshd-config", "config", "--port", port}) require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "port must be zero") }) diff --git a/experimental/ssh/sandbox/export_test.go b/experimental/ssh/sandbox/export_test.go new file mode 100644 index 00000000000..0eb314a8c6a --- /dev/null +++ b/experimental/ssh/sandbox/export_test.go @@ -0,0 +1,16 @@ +package sandbox + +var ( + ProxyStreamForTest = proxyStream + LinuxExecutableForTest = linuxExecutable + ExecutableFromArchiveForTest = executableFromArchive + ReadServerOutputForTest = readServerOutput + ForwardingURLForTest = forwardingURL + PrepareRemoteForTest = prepareRemote + UploadExecutableForTest = uploadExecutable + ConnectProxyForTest = connectProxy + CleanupRemoteDirectoryForTest = cleanupRemoteDirectory + ServerLockScriptForTest = serverLockScript +) + +type RemoteSetupForTest = remoteSetup diff --git a/experimental/ssh/sandbox/identity.go b/experimental/ssh/sandbox/identity.go new file mode 100644 index 00000000000..0ec651219ee --- /dev/null +++ b/experimental/ssh/sandbox/identity.go @@ -0,0 +1,97 @@ +package sandbox + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/pem" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/databricks/cli/experimental/ssh/internal/keys" + "github.com/databricks/cli/experimental/ssh/internal/sshconfig" + "golang.org/x/crypto/ssh" +) + +// Identity contains only paths and public material; private keys remain local. +type Identity struct { + KeyPath string + KnownHostsPath string + HostKeyAlias string + PublicKey []byte +} + +// LocalIdentity creates a reusable key scoped to the workspace, caller and sandbox. +func LocalIdentity(ctx context.Context, workspace, caller, name string) (*Identity, error) { + scope := fmt.Sprintf("sandbox-%x", sha256.Sum256([]byte(workspace+"\x00"+caller+"\x00"+name))) + directory, err := keys.GetLocalSSHKeyPath(ctx, scope, "") + if err != nil { + return nil, err + } + keyPath := filepath.Join(directory, "id_ed25519") + if _, err := os.Stat(directory); errors.Is(err, fs.ErrNotExist) { + if err := createLocalKey(directory); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + private, err := os.ReadFile(keyPath) + if err != nil { + return nil, fmt.Errorf("read local SSH key: %w", err) + } + signer, err := ssh.ParsePrivateKey(private) + if err != nil { + return nil, fmt.Errorf("parse local SSH key: %w", err) + } + if err := os.Chmod(keyPath, 0o600); err != nil { + return nil, err + } + knownHosts, err := sshconfig.GetKnownHostsPath(ctx, scope, "") + if err != nil { + return nil, err + } + return &Identity{KeyPath: keyPath, KnownHostsPath: knownHosts, HostKeyAlias: scope, PublicKey: ssh.MarshalAuthorizedKey(signer.PublicKey())}, nil +} + +func createLocalKey(directory string) error { + if err := os.MkdirAll(filepath.Dir(directory), 0o700); err != nil { + return err + } + temporary, err := os.MkdirTemp(filepath.Dir(directory), ".sandbox-key-*") + if err != nil { + return err + } + defer os.RemoveAll(temporary) + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + encoded, err := ssh.MarshalPrivateKey(private, "") + if err != nil { + return err + } + public, err := ssh.NewPublicKey(private.Public()) + if err != nil { + return err + } + if err := keys.SaveSSHKeyPair(filepath.Join(temporary, "id_ed25519"), pem.EncodeToMemory(encoded), ssh.MarshalAuthorizedKey(public)); err != nil { + return err + } + // Rename a complete directory: concurrent first connections either install + // their key or read the winner, without ever replacing an in-use private key. + if err := os.Rename(temporary, directory); err != nil { + if _, readErr := os.Stat(filepath.Join(directory, "id_ed25519")); readErr != nil { + return fmt.Errorf("install local SSH key: %w", err) + } + } + return nil +} + +func (identity *Identity) PinHostKey(key []byte) error { + return sshconfig.PinHostKey(identity.KnownHostsPath, identity.HostKeyAlias, key) +} diff --git a/experimental/ssh/sandbox/identity_test.go b/experimental/ssh/sandbox/identity_test.go new file mode 100644 index 00000000000..14a7be47d9f --- /dev/null +++ b/experimental/ssh/sandbox/identity_test.go @@ -0,0 +1,59 @@ +package sandbox_test + +import ( + "os" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLocalIdentityConcurrentCreationAndIsolation(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + const count = 12 + identities := make([]*sandbox.Identity, count) + failures := make([]error, count) + var group sync.WaitGroup + for i := range count { + group.Go(func() { + identities[i], failures[i] = sandbox.LocalIdentity(ctx, "https://workspace.test/123", "caller", "sandboxes/test") + }) + } + group.Wait() + for i := range count { + require.NoError(t, failures[i]) + require.NotNil(t, identities[i]) + assert.Equal(t, identities[0], identities[i]) + } + first := identities[0] + for _, scope := range []struct{ workspace, caller, name string }{ + {"https://other.test/123", "caller", "sandboxes/test"}, + {"https://workspace.test/123", "other", "sandboxes/test"}, + {"https://workspace.test/123", "caller", "sandboxes/other"}, + } { + other, err := sandbox.LocalIdentity(ctx, scope.workspace, scope.caller, scope.name) + require.NoError(t, err) + assert.NotEqual(t, first.KeyPath, other.KeyPath) + assert.NotEqual(t, first.PublicKey, other.PublicKey) + } + if runtime.GOOS != "windows" { + stat, err := os.Stat(first.KeyPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), stat.Mode().Perm()) + stat, err = os.Stat(filepath.Dir(first.KeyPath)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), stat.Mode().Perm()) + } + require.NoError(t, first.PinHostKey(first.PublicKey)) + content, err := os.ReadFile(first.KnownHostsPath) + require.NoError(t, err) + assert.Equal(t, first.HostKeyAlias+" "+string(first.PublicKey), string(content)) + require.NoError(t, os.WriteFile(first.KeyPath, []byte("invalid key"), 0o600)) + _, err = sandbox.LocalIdentity(ctx, "https://workspace.test/123", "caller", "sandboxes/test") + require.ErrorContains(t, err, "parse local SSH key") +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/client.go b/experimental/ssh/sandbox/internal/sandboxexec/client.go new file mode 100644 index 00000000000..7dd44d5ba56 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/client.go @@ -0,0 +1,228 @@ +package sandboxexec + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const servicePath = "/databricks.sandbox.SandboxService/" + +type workspaceCredentials struct{ config *config.Config } + +func (c workspaceCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.config.Host, nil) + if err != nil { + return nil, err + } + if err := c.config.Authenticate(request); err != nil { + return nil, err + } + result := make(map[string]string) + for key, value := range auth.WorkspaceIDHeaders(c.config) { + result[strings.ToLower(key)] = value + } + for key, values := range request.Header { + result[strings.ToLower(key)] = strings.Join(values, ",") + } + return result, nil +} +func (workspaceCredentials) RequireTransportSecurity() bool { return true } + +// Client owns the authenticated command-execution channel for one sandbox. +type Client struct { + conn *grpc.ClientConn + name string +} + +func New(w *databricks.WorkspaceClient, name string) (*Client, error) { + host, err := url.Parse(w.Config.Host) + if err != nil || host.Scheme != "https" || host.Hostname() == "" { + return nil, errors.New("sandbox command execution requires an HTTPS workspace URL") + } + address := host.Host + if host.Port() == "" { + address += ":443" + } + conn, err := grpc.NewClient(address, + grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: w.Config.InsecureSkipVerify})), + grpc.WithPerRPCCredentials(workspaceCredentials{config: w.Config})) + if err != nil { + return nil, fmt.Errorf("create sandbox command channel: %w", err) + } + return &Client{conn: conn, name: name}, nil +} + +func (c *Client) Close() error { return c.conn.Close() } + +// Execute leaves the process running when its output stream is closed. Terminate +// must be called separately to stop the process and its descendants. +func (c *Client) Execute(ctx context.Context, commandID, executable string, args []string) (grpc.ServerStreamingClient[ExecuteCommandResponse], error) { + ctx = metadata.AppendToOutgoingContext(ctx, "x-databricks-sandbox-name", c.name) + stream, err := c.conn.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, servicePath+"ExecuteCommand") + if err != nil { + return nil, err + } + if err := stream.SendMsg(&ExecuteCommandRequest{Cmd: new(executable), Args: args, Envs: map[string]string{"TZ": "UTC"}, CommandId: new(commandID)}); err != nil { + return nil, err + } + if err := stream.CloseSend(); err != nil { + return nil, err + } + return &grpc.GenericClientStream[ExecuteCommandRequest, ExecuteCommandResponse]{ClientStream: stream}, nil +} + +func (c *Client) Terminate(ctx context.Context, commandID string) error { + ctx = metadata.AppendToOutgoingContext(ctx, "x-databricks-sandbox-name", c.name) + return c.conn.Invoke(ctx, servicePath+"TerminateCommand", &TerminateCommandRequest{CommandId: new(commandID)}, &TerminateCommandResponse{}) +} + +func (c *Client) attach(ctx context.Context, commandID string, sequence int64) (grpc.ServerStreamingClient[ExecuteCommandResponse], error) { + ctx = metadata.AppendToOutgoingContext(ctx, "x-databricks-sandbox-name", c.name) + stream, err := c.conn.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, servicePath+"AttachCommand") + if err != nil { + return nil, err + } + if err := stream.SendMsg(&AttachCommandRequest{CommandId: new(commandID), LastSequenceNumber: new(sequence)}); err != nil { + return nil, err + } + if err := stream.CloseSend(); err != nil { + return nil, err + } + // AttachCommandResponse has the same process-event and sequence-number fields. + return &grpc.GenericClientStream[AttachCommandRequest, ExecuteCommandResponse]{ClientStream: stream}, nil +} + +// Output renews command-output streams without restarting the remote process. +type Output struct { + client *Client + ctx context.Context + streamCtx context.Context + cancel context.CancelFunc + stream grpc.ServerStreamingClient[ExecuteCommandResponse] + commandID string + executable string + args []string + received bool + sequence int64 + lifetime time.Duration +} + +// Follow reattaches from the last sequence number before the route expires. +// Unconfirmed startup retries reuse commandID so the daemon rejects duplicates. +// Cancel ctx to close the output stream. +func (c *Client) Follow(ctx context.Context, commandID, executable string, args []string, lifetime time.Duration) (*Output, error) { + streamCtx, cancel := context.WithTimeout(ctx, lifetime) + stream, err := c.Execute(streamCtx, commandID, executable, args) + if err != nil { + cancel() + return nil, err + } + return &Output{client: c, ctx: ctx, streamCtx: streamCtx, cancel: cancel, stream: stream, commandID: commandID, executable: executable, args: args, lifetime: lifetime}, nil +} + +func (o *Output) Recv() (*ExecuteCommandResponse, error) { + response, err := o.stream.Recv() + if err == nil { + o.sequence = response.GetSequenceNumber() + o.received = true + return response, nil + } + // A command-output RPC can lose its HTTP/2 transport while the command and + // SSH WebSocket are still healthy. Reattach instead of restarting the command. + reattach := o.ctx.Err() == nil && !errors.Is(err, io.EOF) && + (errors.Is(o.streamCtx.Err(), context.DeadlineExceeded) || retryableOutputError(err)) + o.cancel() + if !reattach { + return nil, err + } + response, attachErr := o.reattach() + if attachErr != nil { + return nil, errors.Join(fmt.Errorf("reattach command output: %w", attachErr), fmt.Errorf("original command-output error: %w", err)) + } + return response, nil +} + +func (o *Output) reattach() (*ExecuteCommandResponse, error) { + // Match the WebSocket resume budget for independent command-output outages. + retryCtx, cancelRetry := context.WithTimeout(o.ctx, time.Minute) + defer cancelRetry() + backoff := 250 * time.Millisecond + execute := false + var err error + for { + if o.ctx.Err() != nil { + return nil, status.FromContextError(o.ctx.Err()).Err() + } + streamCtx, cancel := context.WithTimeout(o.ctx, o.lifetime) + o.streamCtx = streamCtx //nolint:fatcontext // Each RPC derives from the original session context, not the previous RPC. + o.cancel = cancel + // Heartbeats can be 30s apart, so allow the full recovery budget for a response. + stopRetry := context.AfterFunc(retryCtx, cancel) + if execute { + o.stream, err = o.client.Execute(o.streamCtx, o.commandID, o.executable, o.args) + } else { + o.stream, err = o.client.attach(o.streamCtx, o.commandID, o.sequence) + } + var response *ExecuteCommandResponse + if err == nil { + // Streaming RPC failures may surface only on the first receive. + response, err = o.stream.Recv() + } + retryStopped := stopRetry() + if err == nil && retryStopped { + o.sequence = response.GetSequenceNumber() + o.received = true + return response, nil + } + o.cancel() + if o.ctx.Err() != nil { + return nil, status.FromContextError(o.ctx.Err()).Err() + } + if retryCtx.Err() != nil && (err == nil || status.Code(err) == codes.Canceled) { + err = status.FromContextError(retryCtx.Err()).Err() + } + // A lost initial Execute may not have registered the command. Reusing its + // ID is safe: the daemon atomically rejects a duplicate before spawning. + retryStartup := !o.received && ((!execute && status.Code(err) == codes.NotFound) || (execute && status.Code(err) == codes.AlreadyExists)) + if !retryStartup && !retryableOutputError(err) { + return nil, err + } + execute = !o.received && status.Code(err) == codes.NotFound + timer := time.NewTimer(backoff) + select { + case <-retryCtx.Done(): + timer.Stop() + if o.ctx.Err() != nil { + return nil, status.FromContextError(o.ctx.Err()).Err() + } + return nil, err + case <-timer.C: + } + backoff = min(2*backoff, 2*time.Second) + } +} + +func retryableOutputError(err error) bool { + switch status.Code(err) { + case codes.Unavailable, codes.DeadlineExceeded, codes.Internal: + return true + default: + return false + } +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/client_test.go b/experimental/ssh/sandbox/internal/sandboxexec/client_test.go new file mode 100644 index 00000000000..d8a0b570c91 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/client_test.go @@ -0,0 +1,326 @@ +package sandboxexec_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type ( + service any + recordedCall struct { + metadata metadata.MD + command string + } +) + +type commandService struct { + mu sync.Mutex + calls []recordedCall +} + +func (service *commandService) record(ctx context.Context, id string) { + md, _ := metadata.FromIncomingContext(ctx) + service.mu.Lock() + defer service.mu.Unlock() + service.calls = append(service.calls, recordedCall{md.Copy(), id}) +} + +func newCommandTestClient(t *testing.T, server *grpc.Server) *sandboxexec.Client { + t.Helper() + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { + server.ServeHTTP(w, r) + return + } + _, _ = io.WriteString(w, `{}`) + })) + endpoint.EnableHTTP2 = true + endpoint.StartTLS() + t.Cleanup(endpoint.Close) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", WorkspaceID: "123", InsecureSkipVerify: true, HTTPTransport: endpoint.Client().Transport}) + require.NoError(t, err) + client, err := sandboxexec.New(w, "sandboxes/test") + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, client.Close()) }) + return client +} + +func TestAuthenticatedStreamingExecutionAndTermination(t *testing.T) { + implementation := &commandService{} + server := grpc.NewServer() + server.RegisterService(&grpc.ServiceDesc{ + ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*service)(nil), + Streams: []grpc.StreamDesc{{StreamName: "ExecuteCommand", ServerStreams: true, Handler: func(service any, stream grpc.ServerStream) error { + request := &sandboxexec.ExecuteCommandRequest{} + if err := stream.RecvMsg(request); err != nil { + return err + } + implementation.record(stream.Context(), request.GetCommandId()) + if request.Envs["TZ"] != "UTC" { + return errors.New("expected UTC expiry interpretation") + } + if request.GetCmd() != "/remote/databricks" { + return errors.New("unexpected executable") + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Start{Start: &sandboxexec.StartEvent{CommandId: request.CommandId}}}}) + }}}, + Methods: []grpc.MethodDesc{{MethodName: "TerminateCommand", Handler: func(service any, ctx context.Context, decode func(any) error, _ grpc.UnaryServerInterceptor) (any, error) { + request := &sandboxexec.TerminateCommandRequest{} + if err := decode(request); err != nil { + return nil, err + } + implementation.record(ctx, request.GetCommandId()) + return &sandboxexec.TerminateCommandResponse{}, nil + }}}, + }, implementation) + defer server.Stop() + client := newCommandTestClient(t, server) + stream, err := client.Execute(t.Context(), "command-id", "/remote/databricks", []string{"sandbox", "server"}) + require.NoError(t, err) + event, err := stream.Recv() + require.NoError(t, err) + assert.Equal(t, "command-id", event.GetProcessEvent().GetStart().GetCommandId()) + _, err = stream.Recv() + require.ErrorIs(t, err, io.EOF) + require.NoError(t, client.Terminate(t.Context(), "command-id")) + implementation.mu.Lock() + defer implementation.mu.Unlock() + require.Len(t, implementation.calls, 2) + for _, call := range implementation.calls { + assert.Equal(t, []string{"Bearer test-token"}, call.metadata.Get("authorization")) + assert.Equal(t, []string{"123"}, call.metadata.Get("x-databricks-workspace-id")) + assert.Equal(t, []string{"sandboxes/test"}, call.metadata.Get("x-databricks-sandbox-name")) + assert.Equal(t, "command-id", call.command) + } +} + +func TestClientRejectsPlaintext(t *testing.T) { + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: "http://workspace.test", Token: "test-token"}) + require.NoError(t, err) + _, err = sandboxexec.New(w, "sandboxes/test") + require.ErrorContains(t, err, "HTTPS") +} + +func TestOutputRecoversFromLongOutputOnlyOutage(t *testing.T) { + t.Parallel() + const outage = 5 * time.Second + started := sync.OnceValue(time.Now) + var attaches atomic.Int32 + output, starts, _ := newOutputForAttachTest(t, codes.Unavailable, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + assert.Equal(t, int64(7), request.GetLastSequenceNumber()) + if time.Since(started()) < outage { + return status.Error(codes.Unavailable, "command output temporarily unavailable") + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(8))}) + }) + response, err := output.Recv() + require.NoError(t, err) + assert.Equal(t, int64(8), response.GetSequenceNumber()) + assert.GreaterOrEqual(t, time.Since(started()), outage) + assert.Greater(t, attaches.Load(), int32(5)) + _, err = output.Recv() + require.ErrorIs(t, err, io.EOF) + assert.Equal(t, int32(1), starts.Load()) +} + +func TestOutputReattachTiming(t *testing.T) { + t.Parallel() + for _, scenario := range []string{"stalled", "recovered"} { + t.Run(scenario, func(t *testing.T) { + t.Parallel() + var starts, attaches atomic.Int32 + attemptDurations := make(chan time.Duration, 8) + server := grpc.NewServer() + t.Cleanup(server.Stop) + server.RegisterService(&grpc.ServiceDesc{ + ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*service)(nil), + Streams: []grpc.StreamDesc{ + {StreamName: "ExecuteCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.ExecuteCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + starts.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(7))}); err != nil { + return err + } + return status.Error(codes.Unavailable, "command-output transport interrupted") + }}, + {StreamName: "AttachCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + assert.Equal(t, int64(7), request.GetLastSequenceNumber()) + if scenario == "stalled" { + started := time.Now() + <-stream.Context().Done() + attemptDurations <- time.Since(started) + return stream.Context().Err() + } + // Match the production heartbeat delay before the first output. + timer := time.NewTimer(30 * time.Second) + defer timer.Stop() + select { + case <-stream.Context().Done(): + return stream.Context().Err() + case <-timer.C: + } + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(8))}); err != nil { + return err + } + // The recovered stream must also outlive the recovery deadline. + timer.Reset(31 * time.Second) + select { + case <-stream.Context().Done(): + return stream.Context().Err() + case <-timer.C: + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(9))}) + }}, + }, + }, &commandService{}) + client := newCommandTestClient(t, server) + ctx, cancel := context.WithTimeout(t.Context(), 70*time.Second) + defer cancel() + output, err := client.Follow(ctx, "command-id", "databricks", nil, 2*time.Minute) + require.NoError(t, err) + response, err := output.Recv() + require.NoError(t, err) + require.Equal(t, int64(7), response.GetSequenceNumber()) + started := time.Now() + response, err = output.Recv() + if scenario == "recovered" { + require.NoError(t, err) + require.Equal(t, int64(8), response.GetSequenceNumber()) + assert.GreaterOrEqual(t, time.Since(started), 30*time.Second) + response, err = output.Recv() + require.NoError(t, err) + assert.Equal(t, int64(9), response.GetSequenceNumber()) + _, err = output.Recv() + require.ErrorIs(t, err, io.EOF) + } else { + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + select { + case elapsed := <-attemptDurations: + assert.InDelta(t, float64(time.Minute), float64(elapsed), float64(2*time.Second)) + case <-time.After(time.Second): + t.Fatal("stalled attach was not canceled") + } + } + elapsed := time.Since(started) + assert.GreaterOrEqual(t, elapsed, time.Minute) + assert.Less(t, elapsed, 65*time.Second) + assert.Equal(t, int32(1), starts.Load()) + assert.Equal(t, int32(1), attaches.Load()) + }) + } +} + +func TestOutputRecoversUnconfirmedExecution(t *testing.T) { + for _, registered := range []bool{false, true} { + t.Run(fmt.Sprintf("registered=%t", registered), func(t *testing.T) { + t.Parallel() + var executes, attaches, processes atomic.Int32 + rpc := grpc.NewServer() + t.Cleanup(rpc.Stop) + rpc.RegisterService(&grpc.ServiceDesc{ + ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*service)(nil), + Streams: []grpc.StreamDesc{ + {StreamName: "ExecuteCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.ExecuteCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + assert.Equal(t, "command-id", request.GetCommandId()) + assert.Equal(t, "python3", request.GetCmd()) + assert.Equal(t, []string{"-c", "server"}, request.GetArgs()) + attempt := executes.Add(1) + if attempt == 1 { + if registered { + processes.Add(1) + } + return status.Error(codes.Unavailable, "execute transport interrupted") + } + if registered { + return status.Error(codes.AlreadyExists, "command_id is already in use") + } + processes.Add(1) + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(1))}) + }}, + {StreamName: "AttachCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + assert.Equal(t, "command-id", request.GetCommandId()) + assert.Zero(t, request.GetLastSequenceNumber()) + if attaches.Add(1) == 1 { + return status.Error(codes.NotFound, "command not found") + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(1))}) + }}, + }, + }, &commandService{}) + client := newCommandTestClient(t, rpc) + output, err := client.Follow(t.Context(), "command-id", "python3", []string{"-c", "server"}, time.Minute) + require.NoError(t, err) + response, err := output.Recv() + require.NoError(t, err) + assert.Equal(t, int64(1), response.GetSequenceNumber()) + assert.EqualValues(t, 2, executes.Load()) + assert.EqualValues(t, 1, processes.Load()) + expectedAttaches := int32(1) + if registered { + expectedAttaches = 2 + } + assert.Equal(t, expectedAttaches, attaches.Load()) + _, err = output.Recv() + assert.ErrorIs(t, err, io.EOF) + }) + } +} + +func TestOutputDoesNotRestartConfirmedMissingCommand(t *testing.T) { + var attaches atomic.Int32 + output, starts, _ := newOutputForAttachTest(t, codes.Unavailable, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + return status.Error(codes.NotFound, "confirmed command no longer exists") + }) + _, err := output.Recv() + require.Equal(t, codes.NotFound, status.Code(err)) + assert.ErrorContains(t, err, "original command-output error") + assert.EqualValues(t, 1, starts.Load()) + assert.EqualValues(t, 1, attaches.Load()) +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/command.pb.go b/experimental/ssh/sandbox/internal/sandboxexec/command.pb.go new file mode 100644 index 00000000000..4d6317c013e --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/command.pb.go @@ -0,0 +1,739 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v6.33.0 +// source: experimental/ssh/sandbox/internal/sandboxexec/command.proto + +package sandboxexec + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Streaming command messages used by the sandbox SSH client. +type ExecuteCommandRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Cmd *string `protobuf:"bytes,1,opt,name=cmd" json:"cmd,omitempty"` + Args []string `protobuf:"bytes,2,rep,name=args" json:"args,omitempty"` + Envs map[string]string `protobuf:"bytes,4,rep,name=envs" json:"envs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + CommandId *string `protobuf:"bytes,5,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecuteCommandRequest) Reset() { + *x = ExecuteCommandRequest{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecuteCommandRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecuteCommandRequest) ProtoMessage() {} + +func (x *ExecuteCommandRequest) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecuteCommandRequest.ProtoReflect.Descriptor instead. +func (*ExecuteCommandRequest) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{0} +} + +func (x *ExecuteCommandRequest) GetCmd() string { + if x != nil && x.Cmd != nil { + return *x.Cmd + } + return "" +} + +func (x *ExecuteCommandRequest) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *ExecuteCommandRequest) GetEnvs() map[string]string { + if x != nil { + return x.Envs + } + return nil +} + +func (x *ExecuteCommandRequest) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +type ExecuteCommandResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProcessEvent *ProcessEvent `protobuf:"bytes,1,opt,name=process_event,json=processEvent" json:"process_event,omitempty"` + SequenceNumber *int64 `protobuf:"varint,2,opt,name=sequence_number,json=sequenceNumber" json:"sequence_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecuteCommandResponse) Reset() { + *x = ExecuteCommandResponse{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecuteCommandResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecuteCommandResponse) ProtoMessage() {} + +func (x *ExecuteCommandResponse) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecuteCommandResponse.ProtoReflect.Descriptor instead. +func (*ExecuteCommandResponse) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{1} +} + +func (x *ExecuteCommandResponse) GetProcessEvent() *ProcessEvent { + if x != nil { + return x.ProcessEvent + } + return nil +} + +func (x *ExecuteCommandResponse) GetSequenceNumber() int64 { + if x != nil && x.SequenceNumber != nil { + return *x.SequenceNumber + } + return 0 +} + +type ProcessEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Event: + // + // *ProcessEvent_Start + // *ProcessEvent_Data + // *ProcessEvent_End + Event isProcessEvent_Event `protobuf_oneof:"event"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProcessEvent) Reset() { + *x = ProcessEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProcessEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProcessEvent) ProtoMessage() {} + +func (x *ProcessEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProcessEvent.ProtoReflect.Descriptor instead. +func (*ProcessEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{2} +} + +func (x *ProcessEvent) GetEvent() isProcessEvent_Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *ProcessEvent) GetStart() *StartEvent { + if x != nil { + if x, ok := x.Event.(*ProcessEvent_Start); ok { + return x.Start + } + } + return nil +} + +func (x *ProcessEvent) GetData() *DataEvent { + if x != nil { + if x, ok := x.Event.(*ProcessEvent_Data); ok { + return x.Data + } + } + return nil +} + +func (x *ProcessEvent) GetEnd() *EndEvent { + if x != nil { + if x, ok := x.Event.(*ProcessEvent_End); ok { + return x.End + } + } + return nil +} + +type isProcessEvent_Event interface { + isProcessEvent_Event() +} + +type ProcessEvent_Start struct { + Start *StartEvent `protobuf:"bytes,1,opt,name=start,oneof"` +} + +type ProcessEvent_Data struct { + Data *DataEvent `protobuf:"bytes,2,opt,name=data,oneof"` +} + +type ProcessEvent_End struct { + End *EndEvent `protobuf:"bytes,3,opt,name=end,oneof"` +} + +func (*ProcessEvent_Start) isProcessEvent_Event() {} + +func (*ProcessEvent_Data) isProcessEvent_Event() {} + +func (*ProcessEvent_End) isProcessEvent_Event() {} + +type StartEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Pid *int64 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"` + CommandId *string `protobuf:"bytes,2,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartEvent) Reset() { + *x = StartEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartEvent) ProtoMessage() {} + +func (x *StartEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartEvent.ProtoReflect.Descriptor instead. +func (*StartEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{3} +} + +func (x *StartEvent) GetPid() int64 { + if x != nil && x.Pid != nil { + return *x.Pid + } + return 0 +} + +func (x *StartEvent) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +type DataEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Output: + // + // *DataEvent_Stdout + // *DataEvent_Stderr + // *DataEvent_Pty + Output isDataEvent_Output `protobuf_oneof:"output"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DataEvent) Reset() { + *x = DataEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DataEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataEvent) ProtoMessage() {} + +func (x *DataEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataEvent.ProtoReflect.Descriptor instead. +func (*DataEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{4} +} + +func (x *DataEvent) GetOutput() isDataEvent_Output { + if x != nil { + return x.Output + } + return nil +} + +func (x *DataEvent) GetStdout() []byte { + if x != nil { + if x, ok := x.Output.(*DataEvent_Stdout); ok { + return x.Stdout + } + } + return nil +} + +func (x *DataEvent) GetStderr() []byte { + if x != nil { + if x, ok := x.Output.(*DataEvent_Stderr); ok { + return x.Stderr + } + } + return nil +} + +func (x *DataEvent) GetPty() []byte { + if x != nil { + if x, ok := x.Output.(*DataEvent_Pty); ok { + return x.Pty + } + } + return nil +} + +type isDataEvent_Output interface { + isDataEvent_Output() +} + +type DataEvent_Stdout struct { + Stdout []byte `protobuf:"bytes,1,opt,name=stdout,oneof"` +} + +type DataEvent_Stderr struct { + Stderr []byte `protobuf:"bytes,2,opt,name=stderr,oneof"` +} + +type DataEvent_Pty struct { + Pty []byte `protobuf:"bytes,3,opt,name=pty,oneof"` +} + +func (*DataEvent_Stdout) isDataEvent_Output() {} + +func (*DataEvent_Stderr) isDataEvent_Output() {} + +func (*DataEvent_Pty) isDataEvent_Output() {} + +type EndEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + ExitCode *int32 `protobuf:"varint,1,opt,name=exit_code,json=exitCode" json:"exit_code,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EndEvent) Reset() { + *x = EndEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EndEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EndEvent) ProtoMessage() {} + +func (x *EndEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EndEvent.ProtoReflect.Descriptor instead. +func (*EndEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{5} +} + +func (x *EndEvent) GetExitCode() int32 { + if x != nil && x.ExitCode != nil { + return *x.ExitCode + } + return 0 +} + +type TerminateCommandRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId *string `protobuf:"bytes,1,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateCommandRequest) Reset() { + *x = TerminateCommandRequest{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateCommandRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateCommandRequest) ProtoMessage() {} + +func (x *TerminateCommandRequest) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateCommandRequest.ProtoReflect.Descriptor instead. +func (*TerminateCommandRequest) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{6} +} + +func (x *TerminateCommandRequest) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +type TerminateCommandResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateCommandResponse) Reset() { + *x = TerminateCommandResponse{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateCommandResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateCommandResponse) ProtoMessage() {} + +func (x *TerminateCommandResponse) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateCommandResponse.ProtoReflect.Descriptor instead. +func (*TerminateCommandResponse) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{7} +} + +type AttachCommandRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId *string `protobuf:"bytes,1,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + LastSequenceNumber *int64 `protobuf:"varint,2,opt,name=last_sequence_number,json=lastSequenceNumber" json:"last_sequence_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachCommandRequest) Reset() { + *x = AttachCommandRequest{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachCommandRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachCommandRequest) ProtoMessage() {} + +func (x *AttachCommandRequest) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachCommandRequest.ProtoReflect.Descriptor instead. +func (*AttachCommandRequest) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{8} +} + +func (x *AttachCommandRequest) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +func (x *AttachCommandRequest) GetLastSequenceNumber() int64 { + if x != nil && x.LastSequenceNumber != nil { + return *x.LastSequenceNumber + } + return 0 +} + +type AttachCommandResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProcessEvent *ProcessEvent `protobuf:"bytes,1,opt,name=process_event,json=processEvent" json:"process_event,omitempty"` + SequenceNumber *int64 `protobuf:"varint,2,opt,name=sequence_number,json=sequenceNumber" json:"sequence_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachCommandResponse) Reset() { + *x = AttachCommandResponse{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachCommandResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachCommandResponse) ProtoMessage() {} + +func (x *AttachCommandResponse) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachCommandResponse.ProtoReflect.Descriptor instead. +func (*AttachCommandResponse) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{9} +} + +func (x *AttachCommandResponse) GetProcessEvent() *ProcessEvent { + if x != nil { + return x.ProcessEvent + } + return nil +} + +func (x *AttachCommandResponse) GetSequenceNumber() int64 { + if x != nil && x.SequenceNumber != nil { + return *x.SequenceNumber + } + return 0 +} + +var File_experimental_ssh_sandbox_internal_sandboxexec_command_proto protoreflect.FileDescriptor + +const file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc = "" + + "\n" + + ";experimental/ssh/sandbox/internal/sandboxexec/command.proto\x12\x12databricks.sandbox\"\xde\x01\n" + + "\x15ExecuteCommandRequest\x12\x10\n" + + "\x03cmd\x18\x01 \x01(\tR\x03cmd\x12\x12\n" + + "\x04args\x18\x02 \x03(\tR\x04args\x12G\n" + + "\x04envs\x18\x04 \x03(\v23.databricks.sandbox.ExecuteCommandRequest.EnvsEntryR\x04envs\x12\x1d\n" + + "\n" + + "command_id\x18\x05 \x01(\tR\tcommandId\x1a7\n" + + "\tEnvsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x88\x01\n" + + "\x16ExecuteCommandResponse\x12E\n" + + "\rprocess_event\x18\x01 \x01(\v2 .databricks.sandbox.ProcessEventR\fprocessEvent\x12'\n" + + "\x0fsequence_number\x18\x02 \x01(\x03R\x0esequenceNumber\"\xb6\x01\n" + + "\fProcessEvent\x126\n" + + "\x05start\x18\x01 \x01(\v2\x1e.databricks.sandbox.StartEventH\x00R\x05start\x123\n" + + "\x04data\x18\x02 \x01(\v2\x1d.databricks.sandbox.DataEventH\x00R\x04data\x120\n" + + "\x03end\x18\x03 \x01(\v2\x1c.databricks.sandbox.EndEventH\x00R\x03endB\a\n" + + "\x05event\"=\n" + + "\n" + + "StartEvent\x12\x10\n" + + "\x03pid\x18\x01 \x01(\x03R\x03pid\x12\x1d\n" + + "\n" + + "command_id\x18\x02 \x01(\tR\tcommandId\"]\n" + + "\tDataEvent\x12\x18\n" + + "\x06stdout\x18\x01 \x01(\fH\x00R\x06stdout\x12\x18\n" + + "\x06stderr\x18\x02 \x01(\fH\x00R\x06stderr\x12\x12\n" + + "\x03pty\x18\x03 \x01(\fH\x00R\x03ptyB\b\n" + + "\x06output\"'\n" + + "\bEndEvent\x12\x1b\n" + + "\texit_code\x18\x01 \x01(\x05R\bexitCode\"8\n" + + "\x17TerminateCommandRequest\x12\x1d\n" + + "\n" + + "command_id\x18\x01 \x01(\tR\tcommandId\"\x1a\n" + + "\x18TerminateCommandResponse\"g\n" + + "\x14AttachCommandRequest\x12\x1d\n" + + "\n" + + "command_id\x18\x01 \x01(\tR\tcommandId\x120\n" + + "\x14last_sequence_number\x18\x02 \x01(\x03R\x12lastSequenceNumber\"\x87\x01\n" + + "\x15AttachCommandResponse\x12E\n" + + "\rprocess_event\x18\x01 \x01(\v2 .databricks.sandbox.ProcessEventR\fprocessEvent\x12'\n" + + "\x0fsequence_number\x18\x02 \x01(\x03R\x0esequenceNumberBIZGgithub.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + +var ( + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescOnce sync.Once + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescData []byte +) + +func file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP() []byte { + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescOnce.Do(func() { + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc), len(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc))) + }) + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescData +} + +var file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes = make([]protoimpl.MessageInfo, 11) +var file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_goTypes = []any{ + (*ExecuteCommandRequest)(nil), // 0: databricks.sandbox.ExecuteCommandRequest + (*ExecuteCommandResponse)(nil), // 1: databricks.sandbox.ExecuteCommandResponse + (*ProcessEvent)(nil), // 2: databricks.sandbox.ProcessEvent + (*StartEvent)(nil), // 3: databricks.sandbox.StartEvent + (*DataEvent)(nil), // 4: databricks.sandbox.DataEvent + (*EndEvent)(nil), // 5: databricks.sandbox.EndEvent + (*TerminateCommandRequest)(nil), // 6: databricks.sandbox.TerminateCommandRequest + (*TerminateCommandResponse)(nil), // 7: databricks.sandbox.TerminateCommandResponse + (*AttachCommandRequest)(nil), // 8: databricks.sandbox.AttachCommandRequest + (*AttachCommandResponse)(nil), // 9: databricks.sandbox.AttachCommandResponse + nil, // 10: databricks.sandbox.ExecuteCommandRequest.EnvsEntry +} +var file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_depIdxs = []int32{ + 10, // 0: databricks.sandbox.ExecuteCommandRequest.envs:type_name -> databricks.sandbox.ExecuteCommandRequest.EnvsEntry + 2, // 1: databricks.sandbox.ExecuteCommandResponse.process_event:type_name -> databricks.sandbox.ProcessEvent + 3, // 2: databricks.sandbox.ProcessEvent.start:type_name -> databricks.sandbox.StartEvent + 4, // 3: databricks.sandbox.ProcessEvent.data:type_name -> databricks.sandbox.DataEvent + 5, // 4: databricks.sandbox.ProcessEvent.end:type_name -> databricks.sandbox.EndEvent + 2, // 5: databricks.sandbox.AttachCommandResponse.process_event:type_name -> databricks.sandbox.ProcessEvent + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name +} + +func init() { file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_init() } +func file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_init() { + if File_experimental_ssh_sandbox_internal_sandboxexec_command_proto != nil { + return + } + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[2].OneofWrappers = []any{ + (*ProcessEvent_Start)(nil), + (*ProcessEvent_Data)(nil), + (*ProcessEvent_End)(nil), + } + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[4].OneofWrappers = []any{ + (*DataEvent_Stdout)(nil), + (*DataEvent_Stderr)(nil), + (*DataEvent_Pty)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc), len(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc)), + NumEnums: 0, + NumMessages: 11, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_goTypes, + DependencyIndexes: file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_depIdxs, + MessageInfos: file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes, + }.Build() + File_experimental_ssh_sandbox_internal_sandboxexec_command_proto = out.File + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_goTypes = nil + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_depIdxs = nil +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/command.proto b/experimental/ssh/sandbox/internal/sandboxexec/command.proto new file mode 100644 index 00000000000..9ac55440159 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/command.proto @@ -0,0 +1,44 @@ +syntax = "proto2"; +package databricks.sandbox; +option go_package = "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec"; + +// Streaming command messages used by the sandbox SSH client. +message ExecuteCommandRequest { + optional string cmd = 1; + repeated string args = 2; + map envs = 4; + optional string command_id = 5; +} +message ExecuteCommandResponse { + optional ProcessEvent process_event = 1; + optional int64 sequence_number = 2; +} +message ProcessEvent { + oneof event { + StartEvent start = 1; + DataEvent data = 2; + EndEvent end = 3; + } +} +message StartEvent { + optional int64 pid = 1; + optional string command_id = 2; +} +message DataEvent { + oneof output { + bytes stdout = 1; + bytes stderr = 2; + bytes pty = 3; + } +} +message EndEvent { optional int32 exit_code = 1; } +message TerminateCommandRequest { optional string command_id = 1; } +message TerminateCommandResponse {} +message AttachCommandRequest { + optional string command_id = 1; + optional int64 last_sequence_number = 2; +} +message AttachCommandResponse { + optional ProcessEvent process_event = 1; + optional int64 sequence_number = 2; +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/output_test.go b/experimental/ssh/sandbox/internal/sandboxexec/output_test.go new file mode 100644 index 00000000000..cc69a410953 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/output_test.go @@ -0,0 +1,338 @@ +package sandboxexec_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +func TestOutputReattachesWithoutRestartingCommand(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(map[bool]string{false: "rotation", true: "cancellation"}[cancelled], func(t *testing.T) { + var starts, attaches atomic.Int32 + rpc := grpc.NewServer() + rpc.RegisterService(&grpc.ServiceDesc{ + ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*service)(nil), + Streams: []grpc.StreamDesc{ + {StreamName: "ExecuteCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.ExecuteCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + starts.Add(1) + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(7))}); err != nil { + return err + } + <-stream.Context().Done() + return stream.Context().Err() + }}, + {StreamName: "AttachCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + require.Equal(t, "command-id", request.GetCommandId()) + require.Equal(t, int64(7), request.GetLastSequenceNumber()) + md, _ := metadata.FromIncomingContext(stream.Context()) + require.Equal(t, []string{"Bearer test-token"}, md.Get("authorization")) + require.Equal(t, []string{"sandboxes/test"}, md.Get("x-databricks-sandbox-name")) + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(8))}) + }}, + }, + }, &commandService{}) + defer rpc.Stop() + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { + rpc.ServeHTTP(w, r) + return + } + _, _ = io.WriteString(w, `{}`) + })) + endpoint.EnableHTTP2 = true + endpoint.StartTLS() + defer endpoint.Close() + workspace, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", InsecureSkipVerify: true, HTTPTransport: endpoint.Client().Transport}) + require.NoError(t, err) + client, err := sandboxexec.New(workspace, "sandboxes/test") + require.NoError(t, err) + defer client.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + output, err := client.Follow(ctx, "command-id", "databricks", nil, 200*time.Millisecond) + require.NoError(t, err) + first, err := output.Recv() + require.NoError(t, err) + require.Equal(t, int64(7), first.GetSequenceNumber()) + if cancelled { + cancel() + _, err := output.Recv() + require.Equal(t, codes.Canceled, status.Code(err)) + require.Zero(t, attaches.Load()) + return + } + second, err := output.Recv() + require.NoError(t, err) + require.Equal(t, int64(8), second.GetSequenceNumber()) + _, err = output.Recv() + require.ErrorIs(t, err, io.EOF) + require.Equal(t, int32(1), starts.Load()) + require.Equal(t, int32(1), attaches.Load()) + }) + } +} + +func newOutputForAttachTest(t *testing.T, sourceFailure codes.Code, attach grpc.StreamHandler) (*sandboxexec.Output, *atomic.Int32, context.CancelFunc) { + t.Helper() + var starts atomic.Int32 + rpc := grpc.NewServer() + t.Cleanup(rpc.Stop) + rpc.RegisterService(&grpc.ServiceDesc{ + ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*service)(nil), + Streams: []grpc.StreamDesc{ + {StreamName: "ExecuteCommand", ServerStreams: true, Handler: func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.ExecuteCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + starts.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(7))}); err != nil { + return err + } + if sourceFailure != codes.OK { + return status.Error(sourceFailure, "command-output transport interrupted") + } + <-stream.Context().Done() + return stream.Context().Err() + }}, + {StreamName: "AttachCommand", ServerStreams: true, Handler: attach}, + }, + }, &commandService{}) + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { + rpc.ServeHTTP(w, r) + return + } + _, _ = io.WriteString(w, `{}`) + })) + endpoint.EnableHTTP2 = true + endpoint.StartTLS() + t.Cleanup(endpoint.Close) + workspace, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", InsecureSkipVerify: true, HTTPTransport: endpoint.Client().Transport}) + require.NoError(t, err) + client, err := sandboxexec.New(workspace, "sandboxes/test") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, client.Close()) }) + ctx, cancel := context.WithTimeout(t.Context(), 70*time.Second) + t.Cleanup(cancel) + output, err := client.Follow(ctx, "command-id", "databricks", nil, 200*time.Millisecond) + require.NoError(t, err) + first, err := output.Recv() + require.NoError(t, err) + require.Equal(t, int64(7), first.GetSequenceNumber()) + return output, &starts, cancel +} + +func TestOutputRetriesTransientAttachErrors(t *testing.T) { + for _, code := range []codes.Code{codes.Unavailable, codes.DeadlineExceeded, codes.Internal} { + t.Run(code.String(), func(t *testing.T) { + t.Parallel() + var attaches atomic.Int32 + next := make(chan struct{}) + output, starts, _ := newOutputForAttachTest(t, codes.OK, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attempt := attaches.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + sequence := int64(7) + if attempt > 2 { + sequence = 9 + } + assert.Equal(t, sequence, request.GetLastSequenceNumber()) + if attempt == 1 || attempt == 3 { + return status.Error(code, "temporary attach failure") + } + if attempt == 2 { + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(8))}); err != nil { + return err + } + select { + case <-next: + case <-stream.Context().Done(): + return stream.Context().Err() + } + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(9))}); err != nil { + return err + } + <-stream.Context().Done() + return stream.Context().Err() + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(10))}) + }) + response, err := output.Recv() + require.NoError(t, err) + require.Equal(t, int64(8), response.GetSequenceNumber()) + // The successful stream must survive the retry function returning. + close(next) + for _, sequence := range []int64{9, 10} { + response, err = output.Recv() + require.NoError(t, err) + require.Equal(t, sequence, response.GetSequenceNumber()) + } + _, err = output.Recv() + require.ErrorIs(t, err, io.EOF) + require.Equal(t, int32(1), starts.Load()) + require.Equal(t, int32(4), attaches.Load()) + }) + } +} + +func TestOutputReattachesAfterTransportFailure(t *testing.T) { + for _, code := range []codes.Code{codes.Unavailable, codes.DeadlineExceeded, codes.Internal} { + t.Run(code.String(), func(t *testing.T) { + t.Parallel() + var attaches atomic.Int32 + output, starts, _ := newOutputForAttachTest(t, code, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + assert.Equal(t, int64(7), request.GetLastSequenceNumber()) + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{SequenceNumber: new(int64(8))}) + }) + response, err := output.Recv() + require.NoError(t, err) + require.Equal(t, int64(8), response.GetSequenceNumber()) + require.Equal(t, int32(1), starts.Load()) + require.Equal(t, int32(1), attaches.Load()) + }) + } +} + +func TestOutputDoesNotRetryNonTransientAttachErrors(t *testing.T) { + for _, code := range []codes.Code{codes.Unauthenticated, codes.PermissionDenied, codes.NotFound, codes.InvalidArgument, codes.Canceled, codes.OK} { + t.Run(code.String(), func(t *testing.T) { + t.Parallel() + var attaches atomic.Int32 + output, starts, _ := newOutputForAttachTest(t, codes.OK, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + return status.Error(code, "terminal attach result") + }) + _, err := output.Recv() + if code == codes.OK { + require.ErrorIs(t, err, io.EOF) + } else { + require.Equal(t, code, status.Code(err)) + } + require.Equal(t, int32(1), starts.Load()) + require.Equal(t, int32(1), attaches.Load()) + }) + } +} + +func TestOutputCancelsAttachRetry(t *testing.T) { + for _, phase := range []string{"backoff", "receive"} { + t.Run(phase, func(t *testing.T) { + t.Parallel() + var attaches atomic.Int32 + attempted := make(chan struct{}) + output, starts, cancel := newOutputForAttachTest(t, codes.OK, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + if attaches.Add(1) == 1 { + close(attempted) + } + if phase == "receive" { + <-stream.Context().Done() + return stream.Context().Err() + } + return status.Error(codes.Unavailable, "temporary attach failure") + }) + result := make(chan error, 1) + go func() { + _, err := output.Recv() + result <- err + }() + select { + case <-attempted: + case <-time.After(3 * time.Second): + t.Fatal("attach was not attempted") + } + // Let a failed first receive enter its backoff before cancelling. + select { + case err := <-result: + t.Fatalf("receive returned before cancellation: %v", err) + case <-time.After(50 * time.Millisecond): + } + cancel() + select { + case err := <-result: + require.Equal(t, codes.Canceled, status.Code(err)) + case <-time.After(time.Second): + t.Fatal("attach retry did not stop on cancellation") + } + require.Equal(t, int32(1), starts.Load()) + require.Equal(t, int32(1), attaches.Load()) + }) + } +} + +func TestOutputAttachRetryExhaustion(t *testing.T) { + t.Parallel() + for _, code := range []codes.Code{codes.Unavailable, codes.DeadlineExceeded} { + t.Run(code.String(), func(t *testing.T) { + t.Parallel() + var attaches atomic.Int32 + output, starts, _ := newOutputForAttachTest(t, codes.OK, func(_ any, stream grpc.ServerStream) error { + request := new(sandboxexec.AttachCommandRequest) + if err := stream.RecvMsg(request); err != nil { + return err + } + attaches.Add(1) + assert.Equal(t, "command-id", request.GetCommandId()) + assert.Equal(t, int64(7), request.GetLastSequenceNumber()) + return status.Error(code, "attach remains unavailable") + }) + started := time.Now() + result := make(chan error, 1) + go func() { + _, err := output.Recv() + result <- err + }() + select { + case err := <-result: + require.Equal(t, code, status.Code(err)) + case <-time.After(65 * time.Second): + t.Fatal("attach retries did not exhaust within their bound") + } + require.GreaterOrEqual(t, time.Since(started), time.Minute) + require.Equal(t, int32(1), starts.Load()) + require.Greater(t, attaches.Load(), int32(5)) + }) + } +} diff --git a/experimental/ssh/sandbox/native_test.go b/experimental/ssh/sandbox/native_test.go new file mode 100644 index 00000000000..09ecb8a3299 --- /dev/null +++ b/experimental/ssh/sandbox/native_test.go @@ -0,0 +1,263 @@ +package sandbox_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "os" + "os/exec" + "os/user" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/shellquote" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNativeSSHAndSCPThroughWebSocket(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sandbox servers run on Linux") + } + for _, binary := range []string{"ssh", "scp", "sftp", "python3", "ssh-keygen"} { + if _, err := exec.LookPath(binary); err != nil { + t.Skipf("native interoperability test requires %s", binary) + } + } + if _, err := os.Stat("/usr/sbin/sshd"); err != nil { + t.Skip("native interoperability test requires openssh-server") + } + actualHome, err := env.UserHomeDir(t.Context()) + require.NoError(t, err) + home, err := os.MkdirTemp(actualHome, ".sandbox-ssh-test-") //nolint:usetesting // sshd StrictModes rejects authorized_keys under the shared temporary directory. + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(home) }) + identity, err := sandbox.LocalIdentity(env.WithUserHomeDir(t.Context(), home), "https://workspace.test", "caller", "sandboxes/native") + require.NoError(t, err) + script, err := os.ReadFile("setup.py") + require.NoError(t, err) + request, err := json.Marshal(map[string]string{"public_key": string(identity.PublicKey)}) + require.NoError(t, err) + managed := filepath.Join(home, ".databricks", "sandbox-ssh") + require.NoError(t, os.MkdirAll(managed, 0o700)) + for _, directory := range []string{filepath.Dir(managed), managed} { + require.NoError(t, os.Chmod(directory, 0o770)) + } + setup := exec.CommandContext(t.Context(), "python3", "-c", "import os; os.umask(0o002)\n"+string(script), string(request)) + setup.Env = append(os.Environ(), "HOME="+home) + var diagnostics bytes.Buffer + setup.Stderr = &diagnostics + output, err := setup.Output() + require.NoError(t, err, diagnostics.String()) + var remote struct { + Config, SSHD, User string + HostKey string `json:"host_key"` + } + require.NoError(t, json.Unmarshal(output, &remote)) + require.NoError(t, identity.PinHostKey([]byte(remote.HostKey))) + t.Setenv("TZ", "UTC") + executable, err := os.Executable() + require.NoError(t, err) + options := []string{"-F", "/dev/null", "-i", identity.KeyPath, "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + identity.KnownHostsPath, "-o", "HostKeyAlias=" + identity.HostKeyAlias, "-o", "ProxyCommand=" + shellquote.BashArg(executable), "-o", "ConnectTimeout=10"} + account, err := user.Current() + require.NoError(t, err) + destination := account.Username + "@" + identity.HostKeyAlias + ssh := exec.CommandContext(t.Context(), "ssh", append(append([]string{}, options...), destination, "printf 'SSH-byte-stream'; exit 17")...) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", startNativeRelay(t, remote.SSHD, remote.Config)) + output, err = ssh.CombinedOutput() + var exit *exec.ExitError + require.ErrorAs(t, err, &exit, string(output)) + assert.Equal(t, 17, exit.ExitCode(), string(output)) + assert.Contains(t, string(output), "SSH-byte-stream") + pty := exec.CommandContext(t.Context(), "ssh", append(append([]string{}, options...), "-tt", destination, "test -t 0 && printf 'PTY-ready'")...) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", startNativeRelay(t, remote.SSHD, remote.Config)) + output, err = pty.CombinedOutput() + require.NoError(t, err, string(output)) + assert.Contains(t, string(output), "PTY-ready") + + source := filepath.Join(t.TempDir(), "source.bin") + payload := bytes.Repeat([]byte{0, 1, 128, 255}, 10000) + require.NoError(t, os.WriteFile(source, payload, 0o600)) + remoteFile := filepath.Join(t.TempDir(), "remote.bin") + upload := exec.CommandContext(t.Context(), "scp", append(append([]string{}, options...), source, destination+":"+remoteFile)...) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", startNativeRelay(t, remote.SSHD, remote.Config)) + output, err = upload.CombinedOutput() + require.NoError(t, err, string(output)) + copied, err := os.ReadFile(remoteFile) + require.NoError(t, err) + assert.Equal(t, payload, copied) + downloadFile := filepath.Join(t.TempDir(), "download.bin") + download := exec.CommandContext(t.Context(), "scp", append(append([]string{}, options...), destination+":"+remoteFile, downloadFile)...) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", startNativeRelay(t, remote.SSHD, remote.Config)) + output, err = download.CombinedOutput() + require.NoError(t, err, string(output)) + copied, err = os.ReadFile(downloadFile) + require.NoError(t, err) + assert.Equal(t, payload, copied) + // SFTP exercises the subsystem used by modern SCP, even on hosts whose scp + // executable still defaults to the legacy SCP protocol. + batchFile := filepath.Join(t.TempDir(), "batch") + sftpDestination := filepath.Join(t.TempDir(), "sftp.bin") + require.NoError(t, os.WriteFile(batchFile, []byte("put "+source+" "+sftpDestination+"\n"), 0o600)) + transfer := exec.CommandContext(t.Context(), "sftp", append(append([]string{"-b", batchFile}, options...), destination)...) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", startNativeRelay(t, remote.SSHD, remote.Config)) + output, err = transfer.CombinedOutput() + require.NoError(t, err, string(output)) + copied, err = os.ReadFile(sftpDestination) + require.NoError(t, err) + assert.Equal(t, payload, copied) + config, err := os.ReadFile(remote.Config) + require.NoError(t, err) + assert.Contains(t, string(config), "Subsystem sftp internal-sftp") + entries, err := filepath.Glob(filepath.Join(home, ".databricks", "sandbox-ssh", "*", "authorized_keys")) + require.NoError(t, err) + require.Len(t, entries, 1) + contents, err := os.ReadFile(entries[0]) + require.NoError(t, err) + end := strings.Index(string(contents), "\"") + require.GreaterOrEqual(t, end, 0) + expired := `expiry-time="20000101000000" ` + string(identity.PublicKey) + require.NoError(t, os.WriteFile(entries[0], []byte(expired), 0o600)) + rejected := exec.CommandContext(t.Context(), "ssh", append(append([]string{}, options...), destination, "true")...) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", startNativeRelay(t, remote.SSHD, remote.Config)) + output, err = rejected.CombinedOutput() + require.Error(t, err) + assert.Contains(t, string(output), "Permission denied") +} + +func startNativeRelay(t *testing.T, sshd, config string) string { + t.Helper() + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + reader, writer := io.Pipe() + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: sshd, Config: config, ConnectTimeout: time.Minute}, writer) + writer.Close() + }() + t.Cleanup(func() { + cancel() + err := <-finished + if err != nil { + require.ErrorIs(t, err, context.Canceled) + } + }) + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + return "ws://token:" + ready.Token + "@" + listener.Addr().String() + "/ssh" +} + +func TestSetupPrunesOnlyAbandonedSessionDirectories(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sandbox setup runs on Linux") + } + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("requires Python 3") + } + if _, err := exec.LookPath("ssh-keygen"); err != nil { + t.Skip("requires ssh-keygen") + } + if _, err := os.Stat("/usr/sbin/sshd"); err != nil { + t.Skip("requires openssh-server") + } + home := t.TempDir() + require.NoError(t, os.Chmod(home, 0o700)) + ctx := env.WithUserHomeDir(t.Context(), home) + identity, err := sandbox.LocalIdentity(ctx, "https://workspace.test", "caller", "sandboxes/test") + require.NoError(t, err) + script, err := os.ReadFile("setup.py") + require.NoError(t, err) + harness := `import fcntl, json, os, subprocess, sys, time +from pathlib import Path +base = Path.home() / ".databricks" / "sandbox-ssh" +base.mkdir(parents=True) +for name in ("session-old", "session-active", "session-recent", "releases", "persistent-key") + tuple("session-abandoned-" + str(i) for i in range(16)): + directory = base / name + directory.mkdir() + (directory / "keep").write_text("content") + if name != "session-recent": + os.utime(directory, (time.time() - 90000, time.time() - 90000)) +cache = base / "releases" / ("amd64-" + "a" * 64) +cache.mkdir() +executable = cache / "databricks" +executable.write_text("in-use executable") +executable.chmod(0o700) +subprocess.run([sys.executable, "-c", sys.argv[4], "check", str(executable), str(base / "session-active")], check=True) +os.utime(cache, (time.time() - 40 * 86400, time.time() - 40 * 86400)) +server = subprocess.Popen([sys.executable, "-c", sys.argv[3], str(base / "session-active"), "/bin/sleep", "60"]) +try: + with (base / "session-active" / "server.lock").open("a") as lock: + for attempt in range(500): + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.flock(lock, fcntl.LOCK_UN) + except BlockingIOError: + break + assert server.poll() is None + time.sleep(0.01) + else: + raise AssertionError("server did not retain its lease across exec") + os.utime(base / "session-active", (time.time() - 90000, time.time() - 90000)) + # Probe across processes so simultaneous deletion of different old sessions + # fails deterministically, rather than relying on a rare rmtree race. + prefix = '''import fcntl, os, shutil, sys, time +from pathlib import Path +probe = open(Path.home() / ".databricks" / "sandbox-ssh" / ".cleanup-probe", "a") +remove = shutil.rmtree +def checked_remove(path): + fcntl.flock(probe, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + time.sleep(0.01) + remove(path) + finally: + fcntl.flock(probe, fcntl.LOCK_UN) +shutil.rmtree = checked_remove +print("ready", flush=True) +assert sys.stdin.read(1) == "x" +''' + commands = [subprocess.Popen([sys.executable, "-c", prefix + sys.argv[1], json.dumps({"public_key":sys.argv[2]})], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) for _ in range(8)] + try: + for command in commands: + assert command.stdout.readline() == "ready\n" + for command in commands: + command.stdin.write("x") + command.stdin.flush() + for command in commands: + stdout, stderr = command.communicate(timeout=20) + assert command.returncode == 0, (stdout, stderr) + finally: + for command in commands: + if command.poll() is None: + command.kill() + command.wait(timeout=5) + assert not (base / "session-old").exists() + assert not list(base.glob("session-abandoned-*")) + for name in ("session-active", "session-recent", "releases", "persistent-key"): + assert (base / name / "keep").read_text() == "content", name + subprocess.run([sys.executable, "-c", sys.argv[4], "check", str(executable), str(base / "session-active")], check=True) + assert executable.read_text() == "in-use executable" +finally: + server.terminate() + server.wait(timeout=5) +os.utime(base / "session-active", (time.time() - 90000, time.time() - 90000)) +subprocess.run([sys.executable, "-c", sys.argv[1], json.dumps({"public_key":sys.argv[2]})], check=True) +assert not (base / "session-active").exists() +os.utime(cache, (time.time() - 40 * 86400, time.time() - 40 * 86400)) +subprocess.run([sys.executable, "-c", sys.argv[4], "check", str(base / "releases" / ("amd64-" + "b" * 64) / "databricks"), str(base / "session-recent")], check=True) +assert not cache.exists() +` + cacheScript, err := os.ReadFile("cache.py") + require.NoError(t, err) + command := exec.CommandContext(t.Context(), "python3", "-c", harness, string(script), string(identity.PublicKey), sandbox.ServerLockScriptForTest, string(cacheScript)) + command.Env = append(os.Environ(), "HOME="+home) + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) +} diff --git a/experimental/ssh/sandbox/output_linux.go b/experimental/ssh/sandbox/output_linux.go new file mode 100644 index 00000000000..4e1f59f6137 --- /dev/null +++ b/experimental/ssh/sandbox/output_linux.go @@ -0,0 +1,24 @@ +package sandbox + +import ( + "io" + "os" + + "golang.org/x/sys/unix" +) + +func duplicateServerOutput(file *os.File) (io.WriteCloser, error) { + // Command execution owns the pipe; reopening /proc/self/fd requires its + // owner's permissions. Dup retains the inherited descriptor's access. + fd, err := unix.FcntlInt(file.Fd(), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return nil, err + } + // NewFile registers a nonblocking pipe with Go's poller so Close can + // interrupt a heartbeat write when command output stops being consumed. + if err := unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return nil, err + } + return os.NewFile(uintptr(fd), "sandbox command output"), nil +} diff --git a/experimental/ssh/sandbox/output_other.go b/experimental/ssh/sandbox/output_other.go new file mode 100644 index 00000000000..80527bf94de --- /dev/null +++ b/experimental/ssh/sandbox/output_other.go @@ -0,0 +1,13 @@ +//go:build !linux + +package sandbox + +import ( + "errors" + "io" + "os" +) + +func duplicateServerOutput(*os.File) (io.WriteCloser, error) { + return nil, errors.New("server output must be a Linux command-execution pipe") +} diff --git a/experimental/ssh/sandbox/releases.go b/experimental/ssh/sandbox/releases.go new file mode 100644 index 00000000000..418f807bcd0 --- /dev/null +++ b/experimental/ssh/sandbox/releases.go @@ -0,0 +1,159 @@ +package sandbox + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "runtime" + + "github.com/databricks/cli/experimental/ssh/internal/releases" + "github.com/databricks/cli/internal/build" + "github.com/databricks/databricks-sdk-go" + sdk "github.com/databricks/databricks-sdk-go/service/sandbox" +) + +const maxReleaseSize = 256 << 20 + +//go:embed cache.py +var executableCacheScript string + +// executableUploadReader preserves Seek so SDK retries rewind the upload body. +type executableUploadReader struct { + *bytes.Reader +} + +func (executableUploadReader) Close() error { return nil } + +func uploadExecutable(ctx context.Context, w *databricks.WorkspaceClient, name string, setup *remoteSetup, releasesDir string) (string, error) { + architecture := map[string]string{"x86_64": "amd64", "aarch64": "arm64"}[setup.Architecture] + if architecture == "" { + return "", fmt.Errorf("unsupported sandbox architecture %q", setup.Architecture) + } + version := build.GetInfo().Version + cacheID := fmt.Sprintf("%x", sha256.Sum256([]byte(version))) + var data []byte + var err error + if build.GetInfo().IsDevelopment() || releasesDir != "" { + data, err = readLinuxExecutable(ctx, architecture, releasesDir) + if err != nil { + return "", err + } + cacheID = fmt.Sprintf("%x", sha256.Sum256(data)) + } + releaseID := architecture + "-" + cacheID + destination := path.Join(setup.Home, ".databricks", "sandbox-ssh", "releases", releaseID, "databricks") + executable := path.Join(setup.SessionDir, releaseID) + cached, err := executableCacheCommand(ctx, w, name, "check", destination, setup.SessionDir) + if err != nil { + return "", err + } + if cached { + return executable, nil + } + if data == nil { + data, err = readLinuxExecutable(ctx, architecture, releasesDir) + if err != nil { + return "", err + } + } + temporary := path.Join(setup.SessionDir, "databricks.upload") + if err := w.Sandbox.UploadFile(ctx, sdk.UploadFileRequest{Parent: name, Path: temporary, Contents: executableUploadReader{bytes.NewReader(data)}}); err != nil { + return "", fmt.Errorf("upload sandbox SSH executable: %w", err) + } + cached, err = executableCacheCommand(ctx, w, name, "install", destination, setup.SessionDir, temporary, fmt.Sprintf("%x", sha256.Sum256(data))) + if err != nil { + return "", err + } + if !cached { + return "", errors.New("sandbox SSH executable installation did not complete") + } + return executable, nil +} + +func readLinuxExecutable(ctx context.Context, architecture, releasesDir string) ([]byte, error) { + executable, err := linuxExecutable(ctx, architecture, releasesDir) + if err != nil { + return nil, err + } + defer executable.Close() + data, err := io.ReadAll(io.LimitReader(executable, maxReleaseSize+1)) + if err != nil { + return nil, err + } + if len(data) > maxReleaseSize { + return nil, errors.New("linux CLI executable exceeds size limit") + } + return data, nil +} + +func executableCacheCommand(ctx context.Context, w *databricks.WorkspaceClient, name string, args ...string) (bool, error) { + result, err := w.Sandbox.ExecuteCommandSync(ctx, sdk.ExecuteCommandSyncRequest{Name: name, Cmd: "python3", Args: append([]string{"-c", executableCacheScript}, args...)}) + if err != nil { + return false, fmt.Errorf("access sandbox SSH executable cache: %w", err) + } + if result.Status != sdk.ExecuteCommandStatusExecuteCommandStatusCompleted || result.ExitCode != 0 || result.Truncated { + return false, fmt.Errorf("access sandbox SSH executable cache: %s", result.Stderr) + } + var response struct { + Cached bool `json:"cached"` + } + if err := json.Unmarshal([]byte(result.Stdout), &response); err != nil { + return false, fmt.Errorf("read sandbox SSH executable cache result: %w", err) + } + return response.Cached, nil +} + +func linuxExecutable(ctx context.Context, architecture, releasesDir string) (io.ReadCloser, error) { + if releasesDir == "" && runtime.GOOS == "linux" && runtime.GOARCH == architecture { + executable, err := os.Executable() + if err != nil { + return nil, err + } + return os.Open(executable) + } + version := build.GetInfo().Version + if releasesDir == "" && build.GetInfo().IsDevelopment() { + return nil, errors.New("a development build requires --releases-dir with compatible Linux CLI archives") + } + openRelease := releases.OpenPublished + if releasesDir != "" { + openRelease = releases.OpenLocal + } + archive, err := openRelease(ctx, architecture, version, releasesDir) + if err != nil { + return nil, err + } + defer archive.Close() + return executableFromArchive(archive) +} + +func executableFromArchive(archive io.Reader) (io.ReadCloser, error) { + data, err := io.ReadAll(io.LimitReader(archive, maxReleaseSize+1)) + if err != nil { + return nil, err + } + if len(data) > maxReleaseSize { + return nil, errors.New("linux CLI archive exceeds size limit") + } + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("open Linux CLI archive: %w", err) + } + for _, file := range reader.File { + if file.Name == "databricks" { + if file.UncompressedSize64 > maxReleaseSize { + return nil, errors.New("linux CLI executable exceeds size limit") + } + return file.Open() + } + } + return nil, errors.New("linux CLI archive does not contain databricks") +} diff --git a/experimental/ssh/sandbox/releases_test.go b/experimental/ssh/sandbox/releases_test.go new file mode 100644 index 00000000000..fa227431d04 --- /dev/null +++ b/experimental/ssh/sandbox/releases_test.go @@ -0,0 +1,356 @@ +package sandbox_test + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/internal/build" + "github.com/databricks/databricks-sdk-go" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecutableFromArchive(t *testing.T) { + for _, name := range []string{"databricks", "../databricks", "other"} { + t.Run(name, func(t *testing.T) { + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + file, err := writer.Create(name) + require.NoError(t, err) + _, err = file.Write([]byte("executable")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + executable, err := sandbox.ExecutableFromArchiveForTest(&buffer) + if name != "databricks" { + require.ErrorContains(t, err, "does not contain databricks") + return + } + require.NoError(t, err) + defer executable.Close() + data, err := io.ReadAll(executable) + require.NoError(t, err) + assert.Equal(t, "executable", string(data)) + }) + } + _, err := sandbox.ExecutableFromArchiveForTest(bytes.NewBufferString("invalid zip")) + require.ErrorContains(t, err, "open Linux CLI archive") +} + +func TestLinuxExecutableUsesCurrentLinuxBuild(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires a Linux executable") + } + reader, err := sandbox.LinuxExecutableForTest(t.Context(), runtime.GOARCH, "") + require.NoError(t, err) + defer reader.Close() + got, err := io.ReadAll(reader) + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + expected, err := os.ReadFile(executable) + require.NoError(t, err) + assert.Equal(t, expected, got) +} + +func TestLinuxExecutableRequiresDevelopmentArchives(t *testing.T) { + if !build.GetInfo().IsDevelopment() { + t.Skip("requires a development build") + } + reader, err := sandbox.LinuxExecutableForTest(t.Context(), "different-architecture", "") + require.ErrorContains(t, err, "requires --releases-dir") + assert.Nil(t, reader) +} + +// The HTTP fixture executes the actual cache script and stores uploaded files. +func cacheWorkspace(t *testing.T, rejectUpload func([]byte) bool) (*databricks.WorkspaceClient, *atomic.Int32) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("sandbox cache uses Linux file locking") + } + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("cache script requires Python 3") + } + uploads := new(atomic.Int32) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + if r.Method == http.MethodPut { + uploads.Add(1) + destination := r.URL.Query().Get("path") + content, err := io.ReadAll(r.Body) + if !assert.NoError(t, err) { + http.Error(w, "read upload", http.StatusInternalServerError) + return + } + if rejectUpload != nil && rejectUpload(content) { + http.Error(w, `{"error_code":"TEMPORARILY_UNAVAILABLE","message":"retry upload"}`, http.StatusServiceUnavailable) + return + } + if !assert.NoError(t, os.WriteFile(destination, content, 0o600)) { + http.Error(w, "write upload", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusCreated) + return + } + var request struct { + Cmd string `json:"cmd"` + Args []string `json:"args"` + } + if !assert.NoError(t, json.NewDecoder(r.Body).Decode(&request)) { + http.Error(w, "decode command", http.StatusBadRequest) + return + } + command := exec.CommandContext(t.Context(), request.Cmd, request.Args...) + var stderr bytes.Buffer + command.Stderr = &stderr + output, err := command.Output() + exit := 0 + if err != nil { + exit = 1 + } + _ = json.NewEncoder(w).Encode(map[string]any{"status": "EXECUTE_COMMAND_STATUS_COMPLETED", "exit_code": exit, "stdout": string(output), "stderr": stderr.String()}) + })) + t.Cleanup(server.Close) + workspace, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token", RetryTimeoutSeconds: 5}) + require.NoError(t, err) + return workspace, uploads +} + +func writeTestRelease(t *testing.T, directory, content string) { + t.Helper() + filename := "databricks_cli_" + build.GetInfo().Version + "_linux_amd64.zip" + if build.GetInfo().IsDevelopment() { + filename = "databricks_cli_linux_amd64.zip" + } + file, err := os.Create(filepath.Join(directory, filename)) + require.NoError(t, err) + archive := zip.NewWriter(file) + entry, err := archive.Create("databricks") + require.NoError(t, err) + _, err = io.WriteString(entry, content) + require.NoError(t, err) + require.NoError(t, archive.Close()) + require.NoError(t, file.Close()) +} + +func TestUploadExecutableCachesAcrossSessionsAndDistinguishesBuilds(t *testing.T) { + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("cache script requires Python 3") + } + workspace, uploads := cacheWorkspace(t, nil) + releases, home := t.TempDir(), t.TempDir() + writeTestRelease(t, releases, "first-build") + setup := &sandbox.RemoteSetupForTest{Home: home, Architecture: "x86_64", SessionDir: t.TempDir()} + first, err := sandbox.UploadExecutableForTest(t.Context(), workspace, "sandboxes/test", setup, releases) + require.NoError(t, err) + assert.Equal(t, setup.SessionDir, filepath.Dir(first)) + setup.SessionDir = t.TempDir() + second, err := sandbox.UploadExecutableForTest(t.Context(), workspace, "sandboxes/test", setup, releases) + require.NoError(t, err) + assert.Equal(t, setup.SessionDir, filepath.Dir(second)) + assert.NotEqual(t, first, second) + firstInfo, err := os.Stat(first) + require.NoError(t, err) + secondInfo, err := os.Stat(second) + require.NoError(t, err) + assert.True(t, os.SameFile(firstInfo, secondInfo)) + assert.Equal(t, int32(1), uploads.Load()) + contents, err := os.ReadFile(first) + require.NoError(t, err) + assert.Equal(t, "first-build", string(contents)) + info, err := os.Stat(first) + require.NoError(t, err) + if runtime.GOOS != "windows" { + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + } + writeTestRelease(t, releases, "second-build") + third, err := sandbox.UploadExecutableForTest(t.Context(), workspace, "sandboxes/test", setup, releases) + require.NoError(t, err) + assert.NotEqual(t, first, third) + assert.Equal(t, int32(2), uploads.Load()) + setup.Architecture = "riscv64" + _, err = sandbox.UploadExecutableForTest(t.Context(), workspace, "sandboxes/test", setup, releases) + require.ErrorContains(t, err, "unsupported sandbox architecture") +} + +func TestUploadExecutableInstallsAtomicallyDuringConcurrentConnections(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sandbox cache uses Linux file locking") + } + workspace, _ := cacheWorkspace(t, nil) + releases, home := t.TempDir(), t.TempDir() + writeTestRelease(t, releases, "complete-executable") + var group sync.WaitGroup + destinations := make(chan string, 4) + for range 4 { + setup := &sandbox.RemoteSetupForTest{Home: home, Architecture: "x86_64", SessionDir: t.TempDir()} + group.Go(func() { + destination, err := sandbox.UploadExecutableForTest(t.Context(), workspace, "sandboxes/test", setup, releases) + assert.NoError(t, err) + destinations <- destination + }) + } + group.Wait() + close(destinations) + var expected os.FileInfo + paths := make(map[string]bool) + for destination := range destinations { + info, err := os.Stat(destination) + require.NoError(t, err) + if expected == nil { + expected = info + } + assert.True(t, os.SameFile(expected, info)) + paths[destination] = true + data, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, "complete-executable", string(data)) + } + assert.Len(t, paths, 4) +} + +func TestCacheRejectsPartialUploadAndCanRetry(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sandbox cache uses Linux file locking") + } + script, err := os.ReadFile("cache.py") + require.NoError(t, err) + directory := t.TempDir() + target, temporary := filepath.Join(directory, "cache", "databricks"), filepath.Join(directory, "upload") + require.NoError(t, os.WriteFile(temporary, []byte("partial"), 0o600)) + command := exec.CommandContext(t.Context(), "python3", "-c", string(script), "install", target, directory, temporary, "wrong-checksum") + require.Error(t, command.Run()) + _, err = os.Stat(target) + require.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(temporary) + require.ErrorIs(t, err, os.ErrNotExist) + // A subsequent installation uses the same destination after a failed upload. + require.NoError(t, os.WriteFile(temporary, []byte("complete"), 0o600)) + digest := fmt.Sprintf("%x", sha256.Sum256([]byte("complete"))) + command = exec.CommandContext(t.Context(), "python3", "-c", string(script), "install", target, directory, temporary, digest) + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) + data, err := os.ReadFile(target) + require.NoError(t, err) + assert.Equal(t, "complete", string(data)) +} + +func TestUploadExecutableSDKRetryReplaysCompleteBody(t *testing.T) { + attempts := make(chan []byte, 8) + var rejected atomic.Bool + workspace, uploads := cacheWorkspace(t, func(body []byte) bool { + attempts <- body + return !rejected.Swap(true) + }) + releases := t.TempDir() + payload := string(bytes.Repeat([]byte{0, 1, 128, 255}, 10000)) + writeTestRelease(t, releases, payload) + setup := &sandbox.RemoteSetupForTest{Home: t.TempDir(), Architecture: "x86_64", SessionDir: t.TempDir()} + destination, err := sandbox.UploadExecutableForTest(t.Context(), workspace, "sandboxes/test", setup, releases) + require.NoError(t, err) + require.Equal(t, int32(2), uploads.Load()) + for range 2 { + assert.Equal(t, []byte(payload), <-attempts) + } + installed, err := os.ReadFile(destination) + require.NoError(t, err) + assert.Equal(t, []byte(payload), installed) +} + +func TestCachePrunesOldAndExcessReleasesPreservingSessionLeases(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sandbox cache uses Linux file locking") + } + script, err := os.ReadFile("cache.py") + require.NoError(t, err) + harness := `import concurrent.futures, hashlib, json, os, shutil, subprocess, sys, time +from pathlib import Path +base = Path(sys.argv[2]) +releases = base / "releases" +releases.mkdir() + +def release(label, age=0, architecture="amd64"): + directory = releases / (architecture + "-" + hashlib.sha256(label.encode()).hexdigest()) + directory.mkdir() + target = directory / "databricks" + target.write_text(label) + target.chmod(0o700) + os.utime(directory, (time.time() - age, time.time() - age)) + return target + +def check(target, session): + result = subprocess.run([sys.executable, "-c", sys.argv[1], "check", str(target), str(session)], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + assert json.loads(result.stdout) == {"cached": True}, result.stdout + return target + +old = release("old-version", 31 * 86400) +old_dev = release("old-dev-content", 31 * 86400, "arm64") +pinned = release("in-use", 40 * 86400) +session = base / "session-active" +session.mkdir() +check(pinned, session) +os.utime(pinned.parent, (time.time() - 40 * 86400, time.time() - 40 * 86400)) +current = release("current", 40 * 86400) +recent = [release("recent-" + str(i), i + 1) for i in range(12)] +unrelated = releases / "unrelated" +unrelated.mkdir() +(unrelated / "keep").write_text("persistent") +external = base / "external" +external.mkdir() +(external / "keep").write_text("external") +(releases / ("arm64-" + "f" * 64)).symlink_to(external, target_is_directory=True) +starting = base / "session-starting" +starting.mkdir() +check(current, starting) +assert not old.parent.exists() +assert not old_dev.parent.exists() +assert current.read_text() == "current" +assert pinned.read_text() == "in-use" +assert (session / pinned.parent.name).samefile(pinned) +assert [target.exists() for target in recent] == [True] * 7 + [False] * 5 +assert (unrelated / "keep").read_text() == "persistent" +assert (external / "keep").read_text() == "external" + +# A lookup pins an old binary before exec, even while other starts prune. +targets = [] +for i in range(12): + target = release("starting-" + str(i)) + lease_dir = base / ("session-" + str(i)) + lease_dir.mkdir() + check(target, lease_dir) + os.utime(target.parent, (time.time() - 40 * 86400, time.time() - 40 * 86400)) + targets.append((target, lease_dir)) +with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + list(pool.map(lambda item: check(*item), targets)) +for target, lease_dir in targets: + assert target.read_text().startswith("starting-") + assert (lease_dir / target.parent.name).samefile(target) +assert pinned.read_text() == "in-use" + +# Abandoned-session cleanup releases the pin; age pruning can then reclaim it. +shutil.rmtree(session) +check(current, starting) +assert not pinned.parent.exists() +` + command := exec.CommandContext(t.Context(), "python3", "-c", harness, string(script), t.TempDir()) + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) +} diff --git a/experimental/ssh/sandbox/remote.go b/experimental/ssh/sandbox/remote.go new file mode 100644 index 00000000000..9ac79a2f806 --- /dev/null +++ b/experimental/ssh/sandbox/remote.go @@ -0,0 +1,427 @@ +package sandbox + +import ( + "context" + "crypto/tls" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "sync" + "time" + "uuid" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/retries" + sdk "github.com/databricks/databricks-sdk-go/service/sandbox" + "github.com/gorilla/websocket" +) + +//go:embed setup.py +var setupScript string + +type remoteSetup struct { + Home string `json:"home"` + Config string `json:"config"` + SSHD string `json:"sshd"` + User string `json:"user"` + HostKey string `json:"host_key"` + Architecture string `json:"architecture"` + SessionDir string `json:"session_dir"` +} + +// WaitForRunning polls until the sandbox is running, starting it if stopped. +func WaitForRunning(ctx context.Context, w *databricks.WorkspaceClient, name string) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + started := false + progressed := false + for { + sandbox, err := w.Sandbox.GetSandbox(ctx, sdk.GetSandboxRequest{Name: name}) + if err != nil { + return err + } + if sandbox.Status == nil { + return fmt.Errorf("sandbox %s has no runtime status", name) + } + switch sandbox.Status.State { + case sdk.SandboxStateSandboxStateRunning: + return nil + case sdk.SandboxStateSandboxStateStopped: + if started && progressed { + return fmt.Errorf("sandbox %s stopped before becoming ready", name) + } + // Start acknowledgement can precede the observable state transition. + if !started { + if _, err := w.Sandbox.StartSandbox(ctx, sdk.StartSandboxRequest{Name: name}); err != nil { + return err + } + started = true + } + case sdk.SandboxStateSandboxStateStopping, sdk.SandboxStateSandboxStatePending: + progressed = started + default: + return fmt.Errorf("sandbox %s has unsupported state %q", name, sandbox.Status.State) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } +} + +func prepareRemote(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity) (*remoteSetup, error) { + request, err := json.Marshal(map[string]string{"public_key": string(identity.PublicKey)}) + if err != nil { + return nil, err + } + result, err := w.Sandbox.ExecuteCommandSync(ctx, sdk.ExecuteCommandSyncRequest{Name: name, Cmd: "python3", Args: []string{"-c", setupScript, string(request)}}) + if err != nil { + return nil, fmt.Errorf("prepare sandbox SSH configuration: %w", err) + } + if result.Status != sdk.ExecuteCommandStatusExecuteCommandStatusCompleted || result.ExitCode != 0 || result.Truncated { + return nil, fmt.Errorf("sandbox SSH setup failed (%s, exit %d): %s", result.Status, result.ExitCode, result.Stderr) + } + var setup remoteSetup + if err := json.Unmarshal([]byte(result.Stdout), &setup); err != nil { + return nil, fmt.Errorf("read sandbox SSH configuration: %w", err) + } + if setup.User != "sandbox-agent" { + return nil, fmt.Errorf("sandbox SSH requires the sandbox-agent account, got %q", setup.User) + } + if setup.Config == "" || setup.Home == "" || setup.SSHD == "" || setup.User == "" || setup.SessionDir == "" { + return nil, errors.New("incomplete sandbox SSH configuration") + } + if err := identity.PinHostKey([]byte(setup.HostKey)); err != nil { + return nil, err + } + return &setup, nil +} + +// ConnectProxy prepares an invocation's server and carries the SSH stream. It +// never launches a local SSH client or writes diagnostics into the SSH stream. +func ConnectProxy(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity, releasesDir string, input io.ReadCloser, output, diagnostics io.Writer) error { + return connectProxy(ctx, w, name, identity, releasesDir, input, output, diagnostics, 10*time.Minute) +} + +var errServerTermination = errors.New("could not stop sandbox SSH server") + +type forwardingHandshakeError struct { + status int + cause error +} + +func (e *forwardingHandshakeError) Error() string { + return fmt.Sprintf("forwarding WebSocket handshake returned HTTP %d: %v", e.status, e.cause) +} +func (e *forwardingHandshakeError) Unwrap() error { return e.cause } + +func connectProxy(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity, releasesDir string, input io.ReadCloser, output, diagnostics io.Writer, startupTimeout time.Duration) error { + defer input.Close() + deadline := time.Now().Add(startupTimeout) + _, err := retries.Poll(ctx, startupTimeout, func() (*struct{}, *retries.Err) { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil, retries.Halt(context.DeadlineExceeded) + } + // Bound setup and retries without limiting an established SSH session. + err := connectProxyAttempt(ctx, w, name, identity, releasesDir, input, output, diagnostics, remaining) + if err == nil { + return nil, nil + } + handshake, ok := errors.AsType[*forwardingHandshakeError](err) + if errors.Is(err, proxy.ErrConnectFailed) && !errors.Is(err, errServerTermination) && ok { + switch handshake.status { + case http.StatusRequestTimeout, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + // An upgrade error may hide an accepted connection. The failed attempt has + // stopped its server; use a fresh invocation rather than racing a handover. + return nil, retries.Continue(err) + } + } + return nil, retries.Halt(err) + }) + return err +} + +func connectProxyAttempt(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity, releasesDir string, input io.ReadCloser, output, diagnostics io.Writer, startupTimeout time.Duration) (err error) { + ctx, cancelStartup := context.WithCancelCause(ctx) + defer cancelStartup(nil) + defer func() { + if err != nil && ctx.Err() != nil { + err = context.Cause(ctx) + } + }() + // Stop this timer after readiness; it must not limit an established session. + startupTimer := time.AfterFunc(startupTimeout, func() { cancelStartup(context.DeadlineExceeded) }) + defer startupTimer.Stop() + if !cmdio.IsQuiet(ctx) { + fmt.Fprintf(diagnostics, "Waiting for %s to be running...\n", name) + } + if err := WaitForRunning(ctx, w, name); err != nil { + return err + } + setup, err := prepareRemote(ctx, w, name, identity) + if err != nil { + return err + } + defer func() { cleanupRemoteDirectory(ctx, w, name, setup.SessionDir, err, diagnostics) }() + if !cmdio.IsQuiet(ctx) { + fmt.Fprintln(diagnostics, "Preparing sandbox SSH server (checking or uploading the CLI executable)...") + } + binary, err := uploadExecutable(ctx, w, name, setup, releasesDir) + if err != nil { + return err + } + commandClient, err := sandboxexec.New(w, name) + if err != nil { + return err + } + defer commandClient.Close() + commandID := uuid.New().String() + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if stopErr := commandClient.Terminate(cleanupCtx, commandID); stopErr != nil { + log.Warnf(ctx, "Could not stop sandbox SSH server: %v", stopErr) + if err != nil { + err = errors.Join(err, fmt.Errorf("%w: %w", errServerTermination, stopErr)) + } + } + }() + commandCtx, cancel := context.WithCancel(ctx) + defer cancel() + timer := time.AfterFunc(30*time.Second, func() { + cancelStartup(errors.New("timed out waiting for sandbox SSH server readiness")) + }) + defer timer.Stop() + stream, err := commandClient.Follow(commandCtx, commandID, "python3", []string{"-c", serverLockScript, setup.SessionDir, binary, "sandbox", "server", "--sshd-config", setup.Config, "--sshd", setup.SSHD, "--port", "0", "--log-level", "info", "--log-file", path.Join(setup.SessionDir, "server.log")}, 25*time.Minute) + if err != nil { + return fmt.Errorf("start sandbox SSH server: %w", err) + } + ready := make(chan Readiness, 1) + stopped := make(chan error, 1) + go readServerOutput(stream, commandID, ready, stopped, diagnostics) + var readiness Readiness + select { + case result := <-ready: + readiness = result + case err := <-stopped: + if err == nil { + return errors.New("sandbox SSH server exited before connection") + } + return fmt.Errorf("sandbox SSH server exited before readiness: %w", err) + case <-ctx.Done(): + return ctx.Err() + } + if !timer.Stop() || !startupTimer.Stop() { + return errors.New("sandbox SSH startup timed out") + } + // The output stream stays attached so command failure cancels forwarding. + proxyCtx, cancelProxy := context.WithCancelCause(ctx) + defer cancelProxy(nil) + go func() { + select { + case err := <-stopped: + if err != nil { + cancelProxy(fmt.Errorf("sandbox SSH server stopped: %w", err)) + } + case <-proxyCtx.Done(): + } + }() + proxyErr := proxyStream(proxyCtx, input, output, readiness.ResumeVersion == proxy.ResumeProtocolVersion, + func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + endpoint, err := forwardingURL(w.Config.Host, name, readiness.Port, request) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set(RelayTokenHeader, readiness.Token) + for key, value := range auth.WorkspaceIDHeaders(w.Config) { + req.Header.Set(key, value) + } + if err := w.Config.Authenticate(req); err != nil { + return nil, err + } + dialer := *websocket.DefaultDialer + dialer.HandshakeTimeout = 30 * time.Second + dialer.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: w.Config.InsecureSkipVerify} + conn, response, err := dialer.DialContext(ctx, endpoint, req.Header) + if response != nil { + response.Body.Close() + if err != nil { + err = &forwardingHandshakeError{status: response.StatusCode, cause: err} + } + } + if err != nil && request.Reattach && response != nil && (response.StatusCode == http.StatusConflict || response.StatusCode == http.StatusGone) { + return nil, errors.Join(proxy.ErrReattachRejected, err) + } + return conn, err + }) + // Seal completion before the server-output watcher can report a later exit. + cancelProxy(nil) + if cause := context.Cause(proxyCtx); cause != nil && !errors.Is(cause, context.Canceled) { + return cause + } + return proxyErr +} + +func forwardingURL(host, name string, port int, dial proxy.DialRequest) (string, error) { + endpoint, err := url.Parse(host) + if err != nil || endpoint.Scheme != "https" { + return "", errors.New("port forwarding requires an HTTPS workspace URL") + } + if !strings.HasPrefix(name, "sandboxes/") || strings.ContainsAny(strings.TrimPrefix(name, "sandboxes/"), "/?#") { + return "", fmt.Errorf("invalid sandbox resource name %q", name) + } + endpoint.Scheme = "wss" + endpoint.Path = fmt.Sprintf("/sandbox-proxy/%s/ports/%d/ssh", name, port) + query := url.Values{"id": {dial.ConnID}} + if dial.ResumeCapable { + query.Set(proxy.ResumeVersionParameter, strconv.Itoa(proxy.ResumeProtocolVersion)) + query.Set("delivered", strconv.FormatInt(dial.Delivered, 10)) + if dial.Reattach { + query.Set("reattach", "1") + } + } + endpoint.RawQuery = query.Encode() + return endpoint.String(), nil +} + +func readServerOutput(stream interface { + Recv() (*sandboxexec.ExecuteCommandResponse, error) +}, commandID string, ready chan<- Readiness, stopped chan<- error, diagnostics io.Writer, +) { + var stdout strings.Builder + announced := false + for { + response, err := stream.Recv() + if err != nil { + stopped <- err + return + } + event := response.GetProcessEvent() + if start := event.GetStart(); start != nil && start.GetCommandId() != commandID { + stopped <- errors.New("sandbox returned an unexpected command ID") + return + } + if end := event.GetEnd(); end != nil { + if end.ExitCode == nil { + stopped <- errors.New("server exited without an exit status") + return + } + if announced && end.GetExitCode() == 0 { + // Let the forwarding stream drain its final SSH bytes and close normally. + stopped <- nil + return + } + stopped <- fmt.Errorf("server exited with status %d", end.GetExitCode()) + return + } + if data := event.GetData(); data != nil { + if !announced { + if _, err := diagnostics.Write(data.GetStderr()); err != nil { + stopped <- err + return + } + stdout.Write(data.GetStdout()) + if stdout.Len() > 4096 { + stopped <- errors.New("sandbox server readiness output is too large") + return + } + if strings.Contains(stdout.String(), "\n") { + var result Readiness + if err := json.Unmarshal([]byte(strings.SplitN(stdout.String(), "\n", 2)[0]), &result); err != nil { + stopped <- err + return + } + if !validForwardPort(result.Port) { + stopped <- fmt.Errorf("invalid sandbox relay port %d", result.Port) + return + } + if result.Token == "" { + stopped <- errors.New("sandbox relay readiness has no invocation token") + return + } + ready <- result + announced = true + } + } + } + } +} + +func cleanupRemoteDirectory(ctx context.Context, w *databricks.WorkspaceClient, name, directory string, sessionErr error, diagnostics io.Writer) { + if sessionErr != nil && !errors.Is(context.Cause(ctx), context.Canceled) { + fmt.Fprintf(diagnostics, "Sandbox SSH session files retained at %s in the sandbox for diagnosis\n", directory) + return + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + _, err := w.Sandbox.ExecuteCommandSync(ctx, sdk.ExecuteCommandSyncRequest{Name: name, Cmd: "rm", Args: []string{"-rf", "--", directory}}) + if err != nil { + log.Debugf(ctx, "Could not remove sandbox SSH session files: %v", err) + } +} + +// A read from inherited stdin can remain blocked after Close on Unix. Keep that +// read outside the shared proxy's wait group so a closed WebSocket can end the +// standalone proxy command. Process exit reclaims any remaining stdin read. +func proxyStream(ctx context.Context, input io.ReadCloser, output io.Writer, resumable bool, dial func(context.Context, proxy.DialRequest) (*websocket.Conn, error)) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + connected := make(chan struct{}) + ready := sync.OnceFunc(func() { close(connected) }) + defer func() { + select { + case <-connected: + input.Close() + default: + } + }() + go func() { + select { + case <-connected: + _, err := io.Copy(writer, input) + writer.CloseWithError(err) + case <-ctx.Done(): + } + }() + // Keep stdin untouched until the upgrade succeeds, so a failed initial + // connection can retry with a fresh remote server without losing SSH bytes. + connect := func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + conn, err := dial(ctx, request) + if err == nil { + ready() + } + return conn, err + } + handover := time.NewTicker(25 * time.Minute) + defer handover.Stop() + return proxy.RunClientProxy(ctx, reader, output, func() <-chan time.Time { return handover.C }, 30*time.Second, resumable, connect) +} + +// Keep the session lease open in the server, including after the local CLI dies. +const serverLockScript = `import fcntl, os, sys +lock = open(os.path.join(sys.argv[1], "server.lock"), "a") +fcntl.flock(lock, fcntl.LOCK_EX) +os.set_inheritable(lock.fileno(), True) +os.execv(sys.argv[2], sys.argv[2:]) +` diff --git a/experimental/ssh/sandbox/remote_integration_test.go b/experimental/ssh/sandbox/remote_integration_test.go new file mode 100644 index 00000000000..f07da497f98 --- /dev/null +++ b/experimental/ssh/sandbox/remote_integration_test.go @@ -0,0 +1,376 @@ +package sandbox_test + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type execService any + +type remoteFixture struct { + mode string + forwardStatus, forwardFailures int + terminateError bool + resumeVersion int + publicKey string + connected chan struct{} + outputEnded chan struct{} + mu sync.Mutex + terminated, removed, uploaded bool + authenticated bool + commandID string + dialQueries []url.Values +} + +func (fixture *remoteFixture) execute(_ any, stream grpc.ServerStream) error { + request := &sandboxexec.ExecuteCommandRequest{} + if err := stream.RecvMsg(request); err != nil { + return err + } + fixture.mu.Lock() + fixture.commandID = request.GetCommandId() + fixture.mu.Unlock() + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Start{Start: &sandboxexec.StartEvent{CommandId: request.CommandId}}}}); err != nil { + return err + } + switch fixture.mode { + case "before-ready": + case "cancel", "startup-timeout": + <-stream.Context().Done() + return stream.Context().Err() + case "after-ready", "clean-close", "slow-session", "clean-output-first", "resume-dial", "resume-gone", "resume-conflict": + readiness := "{\"port\":40000,\"token\":\"invocation-token\"}\n" + if fixture.resumeVersion != 0 { + readiness = fmt.Sprintf("{\"port\":40000,\"token\":\"invocation-token\",\"resume_version\":%d}\n", fixture.resumeVersion) + } + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stdout{Stdout: []byte(readiness)}}}}}); err != nil { + return err + } + select { + case <-fixture.connected: + case <-stream.Context().Done(): + return stream.Context().Err() + } + if fixture.mode == "clean-close" || fixture.mode == "slow-session" || fixture.mode == "resume-gone" || fixture.mode == "resume-conflict" { + <-stream.Context().Done() + return stream.Context().Err() + } + } + if fixture.mode == "clean-output-first" { + err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_End{End: &sandboxexec.EndEvent{ExitCode: proto.Int32(0)}}}}) + close(fixture.outputEnded) + return err + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_End{End: &sandboxexec.EndEvent{ExitCode: proto.Int32(17)}}}}) +} + +func (fixture *remoteFixture) terminate(_ any, _ context.Context, decode func(any) error, _ grpc.UnaryServerInterceptor) (any, error) { + request := &sandboxexec.TerminateCommandRequest{} + if err := decode(request); err != nil { + return nil, err + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + if request.GetCommandId() != fixture.commandID { + return nil, errors.New("termination used a different command ID") + } + fixture.terminated = true + if fixture.terminateError { + return nil, status.Error(codes.Unavailable, "termination failed") + } + return &sandboxexec.TerminateCommandResponse{}, nil +} + +func (fixture *remoteFixture) serveHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/.well-known/databricks-config": + _, _ = io.WriteString(w, `{}`) + case r.URL.Path == "/api/2.0/sandboxes/test": + _, _ = io.WriteString(w, `{"status":{"state":"SANDBOX_STATE_RUNNING"}}`) + case strings.HasPrefix(r.URL.Path, "/api/2.0/sandbox-files/"): + fixture.mu.Lock() + fixture.uploaded = true + fixture.mu.Unlock() + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(http.StatusCreated) + case strings.HasSuffix(r.URL.Path, "/exec-sync"): + var request struct { + Cmd string `json:"cmd"` + Args []string `json:"args"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + result := map[string]any{"status": "EXECUTE_COMMAND_STATUS_COMPLETED"} + switch request.Cmd { + case "python3": + if len(request.Args) > 2 && (request.Args[2] == "check" || request.Args[2] == "install") { + result["stdout"] = fmt.Sprintf(`{"cached":%t}`, request.Args[2] == "install") + break + } + setup, _ := json.Marshal(map[string]string{"home": "/home/sandbox-agent", "config": "/home/sandbox-agent/session/sshd_config", "sshd": "/usr/sbin/sshd", "user": "sandbox-agent", "host_key": fixture.publicKey, "session_dir": "/home/sandbox-agent/session", "architecture": "x86_64"}) + result["stdout"] = string(setup) + case "rm": + fixture.mu.Lock() + fixture.removed = true + fixture.mu.Unlock() + } + _ = json.NewEncoder(w).Encode(result) + case strings.HasPrefix(r.URL.Path, "/sandbox-proxy/"): + if r.Header.Get(sandbox.RelayTokenHeader) != "invocation-token" || r.Header.Get("Authorization") != "Bearer test-token" || r.Header.Get("X-Databricks-Workspace-Id") != "123" { + http.Error(w, "missing forwarding credentials", http.StatusUnauthorized) + return + } + fixture.mu.Lock() + fixture.authenticated = true + fixture.dialQueries = append(fixture.dialQueries, r.URL.Query()) + attempt := len(fixture.dialQueries) + fixture.mu.Unlock() + if attempt <= fixture.forwardFailures { + http.Error(w, "forwarding rejected", fixture.forwardStatus) + return + } + if fixture.mode == "resume-dial" { + http.Error(w, "connection rejected", http.StatusGone) + return + } + if r.URL.Query().Get("reattach") == "1" { + switch fixture.mode { + case "resume-gone": + http.Error(w, "session expired", http.StatusGone) + return + case "resume-conflict": + http.Error(w, "session unavailable", http.StatusConflict) + return + } + } + upgrader := websocket.Upgrader{} + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + _ = conn.WriteMessage(websocket.BinaryMessage, []byte("SSH-2.0-test\r\n")) + close(fixture.connected) + if fixture.mode == "resume-gone" || fixture.mode == "resume-conflict" { + return + } + if fixture.mode == "slow-session" { + time.Sleep(1500 * time.Millisecond) + } + if fixture.mode == "clean-output-first" { + <-fixture.outputEnded + // Deliver the clean command exit before closing the forwarding connection. + time.Sleep(200 * time.Millisecond) + } + if fixture.mode == "clean-close" || fixture.mode == "slow-session" || fixture.mode == "clean-output-first" { + _ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + return + } + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + default: + http.NotFound(w, r) + } +} + +func TestConnectProxyCleansUpAfterCompletionFailureAndCancellation(t *testing.T) { + for _, tc := range []struct { + name, mode string + resumeVersion int + quiet bool + forwardStatus, forwardFailures int + wantForwardError bool + terminateError bool + }{ + {name: "forwarding-503", mode: "clean-close", forwardStatus: http.StatusServiceUnavailable, forwardFailures: 2}, + {name: "forwarding-429", mode: "clean-close", forwardStatus: http.StatusTooManyRequests, forwardFailures: 1}, + {name: "forwarding-503-stop-failed", mode: "clean-close", forwardStatus: http.StatusServiceUnavailable, forwardFailures: 1, wantForwardError: true, terminateError: true}, + {name: "forwarding-403", mode: "clean-close", forwardStatus: http.StatusForbidden, forwardFailures: 1, wantForwardError: true}, + {name: "before-ready", mode: "before-ready"}, + {name: "after-ready", mode: "after-ready"}, + {name: "cancel", mode: "cancel"}, + {name: "clean-close", mode: "clean-close"}, + {name: "quiet-clean-close", mode: "clean-close", quiet: true}, + {name: "startup-timeout", mode: "startup-timeout"}, + {name: "slow-session", mode: "slow-session"}, + {name: "clean-output-first", mode: "clean-output-first"}, + {name: "unsupported-resume-v1", mode: "clean-close", resumeVersion: 1}, + {name: "unsupported-resume-future", mode: "clean-close", resumeVersion: proxy.ResumeProtocolVersion + 1}, + {name: "resume-initial-dial", mode: "resume-dial", resumeVersion: proxy.ResumeProtocolVersion}, + {name: "resume-reattach-gone", mode: "resume-gone", resumeVersion: proxy.ResumeProtocolVersion}, + {name: "resume-reattach-conflict", mode: "resume-conflict", resumeVersion: proxy.ResumeProtocolVersion}, + } { + t.Run(tc.name, func(t *testing.T) { + mode := tc.mode + home := t.TempDir() + ctx := env.WithUserHomeDir(t.Context(), home) + if tc.quiet { + ctx = cmdio.WithQuiet(ctx) + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + identity, err := sandbox.LocalIdentity(ctx, "https://workspace.test", "caller", "sandboxes/test") + require.NoError(t, err) + fixture := &remoteFixture{mode: mode, terminateError: tc.terminateError, forwardStatus: tc.forwardStatus, forwardFailures: tc.forwardFailures, resumeVersion: tc.resumeVersion, publicKey: string(identity.PublicKey), connected: make(chan struct{}), outputEnded: make(chan struct{})} + grpcServer := grpc.NewServer() + grpcServer.RegisterService(&grpc.ServiceDesc{ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*execService)(nil), Streams: []grpc.StreamDesc{{StreamName: "ExecuteCommand", ServerStreams: true, Handler: fixture.execute}}, Methods: []grpc.MethodDesc{{MethodName: "TerminateCommand", Handler: fixture.terminate}}}, fixture) + defer grpcServer.Stop() + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { + grpcServer.ServeHTTP(w, r) + return + } + fixture.serveHTTP(w, r) + })) + endpoint.EnableHTTP2 = true + endpoint.StartTLS() + defer endpoint.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", WorkspaceID: "123", InsecureSkipVerify: true, HTTPTransport: endpoint.Client().Transport}) + require.NoError(t, err) + filename := "databricks_cli_" + build.GetInfo().Version + "_linux_amd64.zip" + if build.GetInfo().IsDevelopment() { + filename = "databricks_cli_linux_amd64.zip" + } + var buffer bytes.Buffer + archive := zip.NewWriter(&buffer) + file, err := archive.Create("databricks") + require.NoError(t, err) + _, err = file.Write([]byte("executable")) + require.NoError(t, err) + require.NoError(t, archive.Close()) + require.NoError(t, os.WriteFile(filepath.Join(home, filename), buffer.Bytes(), 0o600)) + if mode == "cancel" { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + defer cancel() + timer := time.AfterFunc(time.Second, cancel) + defer timer.Stop() + } + input, writer := io.Pipe() + defer input.Close() + defer writer.Close() + started := time.Now() + startupTimeout := 10 * time.Minute + if mode == "startup-timeout" || mode == "slow-session" { + startupTimeout = time.Second + } + var diagnostics bytes.Buffer + err = sandbox.ConnectProxyForTest(ctx, w, "sandboxes/test", identity, home, input, io.Discard, &diagnostics, startupTimeout) + if (mode == "clean-close" || mode == "slow-session" || mode == "clean-output-first") && !tc.wantForwardError { + require.NoError(t, err) + } else { + require.Error(t, err) + } + if tc.wantForwardError { + assert.ErrorContains(t, err, fmt.Sprintf("HTTP %d", tc.forwardStatus)) + } + switch mode { + case "before-ready": + assert.Contains(t, err.Error(), "exited before readiness") + case "after-ready": + assert.Contains(t, err.Error(), "server stopped") + case "cancel": + assert.ErrorIs(t, err, context.Canceled) + case "startup-timeout": + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(started), 5*time.Second) + case "slow-session": + assert.Greater(t, time.Since(started), startupTimeout) + case "resume-dial": + assert.ErrorIs(t, err, proxy.ErrConnectFailed) + assert.NotErrorIs(t, err, proxy.ErrReattachRejected) + case "resume-gone", "resume-conflict": + assert.ErrorIs(t, err, proxy.ErrWebsocketDropped) + assert.ErrorIs(t, err, proxy.ErrReattachRejected) + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + if mode == "after-ready" || mode == "clean-close" || tc.resumeVersion == proxy.ResumeProtocolVersion { + assert.True(t, fixture.authenticated) + } + if fixture.authenticated { + if mode == "resume-gone" || mode == "resume-conflict" { + require.Len(t, fixture.dialQueries, 2) + assert.Equal(t, url.Values{ + "id": {fixture.dialQueries[0].Get("id")}, + proxy.ResumeVersionParameter: {"2"}, + "delivered": {strconv.Itoa(len("SSH-2.0-test\r\n"))}, + "reattach": {"1"}, + }, fixture.dialQueries[1]) + } else { + expected := 1 + if !tc.wantForwardError { + expected += tc.forwardFailures + } + require.Len(t, fixture.dialQueries, expected) + for i, query := range fixture.dialQueries[1:] { + if tc.forwardFailures > 0 { + assert.NotEqual(t, fixture.dialQueries[i].Get("id"), query.Get("id")) + } else { + assert.Equal(t, fixture.dialQueries[0], query) + } + } + } + query := fixture.dialQueries[0] + assert.NotEmpty(t, query.Get("id")) + assert.False(t, query.Has("reattach")) + if tc.resumeVersion == proxy.ResumeProtocolVersion { + assert.Equal(t, "2", query.Get(proxy.ResumeVersionParameter)) + assert.Equal(t, "0", query.Get("delivered")) + } else { + assert.False(t, query.Has(proxy.ResumeVersionParameter)) + assert.False(t, query.Has("delivered")) + } + } + assert.True(t, fixture.uploaded) + assert.True(t, fixture.terminated, "server command must be terminated independently of stream cancellation") + remove := err == nil || mode == "cancel" + assert.Equal(t, remove, fixture.removed, "preserve session logs on failures; clean up successful or cancelled sessions") + if !remove { + assert.Contains(t, diagnostics.String(), "session files retained at /home/sandbox-agent/session") + } + if tc.quiet { + assert.NotContains(t, diagnostics.String(), "Waiting for") + assert.NotContains(t, diagnostics.String(), "checking or uploading") + } else { + assert.Contains(t, diagnostics.String(), "Waiting for sandboxes/test to be running") + assert.Contains(t, diagnostics.String(), "checking or uploading the CLI executable") + } + }) + } +} diff --git a/experimental/ssh/sandbox/remote_test.go b/experimental/ssh/sandbox/remote_test.go new file mode 100644 index 00000000000..1de97d1f141 --- /dev/null +++ b/experimental/ssh/sandbox/remote_test.go @@ -0,0 +1,350 @@ +package sandbox_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/env" + "github.com/gorilla/websocket" + + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWaitForRunningLifecycle(t *testing.T) { + for _, states := range [][]string{ + {"RUNNING"}, {"STOPPED", "RUNNING"}, {"STOPPING", "STOPPED", "PENDING", "RUNNING"}, {"PENDING", "RUNNING"}, {"UNSPECIFIED"}, {"STOPPED", "STOPPED", "RUNNING"}, {"STOPPED", "PENDING", "STOPPED"}, + } { + t.Run(strings.Join(states, "-"), func(t *testing.T) { + index, starts := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/sandboxes/test", strings.TrimSuffix(r.URL.Path, "/start")) + assert.Equal(t, "123", r.Header.Get("X-Databricks-Workspace-Id")) + if r.Method == http.MethodPost { + starts++ + _, _ = io.WriteString(w, `{}`) + return + } + state := states[min(index, len(states)-1)] + index++ + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]string{"state": "SANDBOX_STATE_" + state}}) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token", WorkspaceID: "123"}) + require.NoError(t, err) + err = sandbox.WaitForRunning(t.Context(), w, "sandboxes/test") + if states[len(states)-1] != "RUNNING" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + if strings.Contains(strings.Join(states, "-"), "STOPPED") { + assert.Equal(t, 1, starts) + } else { + assert.Zero(t, starts) + } + }) + } +} + +type responseStream struct { + events []*sandboxexec.ExecuteCommandResponse +} + +func TestCleanupRemoteDirectoryRetainsFailureLogs(t *testing.T) { + for _, tc := range []struct { + name string + cause error + sessionErr error + remove bool + }{ + {name: "success", remove: true}, + {name: "failure", sessionErr: errors.New("server failed")}, + {name: "user-cancellation", cause: context.Canceled, sessionErr: context.Canceled, remove: true}, + {name: "startup-timeout", cause: context.DeadlineExceeded, sessionErr: context.Canceled}, + {name: "readiness-timeout", cause: errors.New("timed out waiting for sandbox SSH server readiness"), sessionErr: context.Canceled}, + } { + t.Run(tc.name, func(t *testing.T) { + var removed atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/sandbox-exec/sandboxes/test/exec-sync", r.URL.Path) + var request struct { + Cmd string `json:"cmd"` + Args []string `json:"args"` + } + if !assert.NoError(t, json.NewDecoder(r.Body).Decode(&request)) { + w.WriteHeader(http.StatusBadRequest) + return + } + assert.Equal(t, "rm", request.Cmd) + assert.Equal(t, []string{"-rf", "--", "/home/sandbox-agent/session"}, request.Args) + removed.Store(true) + _, _ = io.WriteString(w, `{"status":"EXECUTE_COMMAND_STATUS_COMPLETED"}`) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + ctx, cancel := context.WithCancelCause(t.Context()) + defer cancel(nil) + if tc.cause != nil { + cancel(tc.cause) + } + var diagnostics bytes.Buffer + sandbox.CleanupRemoteDirectoryForTest(ctx, w, "sandboxes/test", "/home/sandbox-agent/session", tc.sessionErr, &diagnostics) + assert.Equal(t, tc.remove, removed.Load()) + if tc.remove { + assert.Empty(t, diagnostics.String()) + } else { + assert.Contains(t, diagnostics.String(), "session files retained at /home/sandbox-agent/session") + } + }) + } +} + +func (stream *responseStream) Recv() (*sandboxexec.ExecuteCommandResponse, error) { + if len(stream.events) == 0 { + return nil, io.EOF + } + event := stream.events[0] + stream.events = stream.events[1:] + return event, nil +} + +func TestReadServerOutput(t *testing.T) { + response := func(data string) *sandboxexec.ExecuteCommandResponse { + return &sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stdout{Stdout: []byte(data)}}}}} + } + for _, tc := range []struct { + name string + chunks []string + port int + errorText string + }{ + {"fragmented", []string{"{\"port\":4", "0000,\"token\":\"invocation-token\"}\n"}, 40000, "EOF"}, + {"invalid-json", []string{"hello\n"}, 0, "invalid character"}, + {"invalid-port", []string{"{\"port\":22}\n"}, 0, "invalid sandbox relay port"}, + {"missing-token", []string{"{\"port\":40000}\n"}, 0, "no invocation token"}, + {"empty-token", []string{"{\"port\":40000,\"token\":\"\"}\n"}, 0, "no invocation token"}, + {"too-large", []string{strings.Repeat("x", 4097)}, 0, "too large"}, + } { + t.Run(tc.name, func(t *testing.T) { + stream := &responseStream{} + for _, chunk := range tc.chunks { + stream.events = append(stream.events, response(chunk)) + } + ready := make(chan sandbox.Readiness, 1) + stopped := make(chan error, 1) + sandbox.ReadServerOutputForTest(stream, "command", ready, stopped, io.Discard) + require.ErrorContains(t, <-stopped, tc.errorText) + if tc.port != 0 { + assert.Equal(t, tc.port, (<-ready).Port) + } else { + assert.Empty(t, ready) + } + }) + } + ready := make(chan sandbox.Readiness, 1) + stopped := make(chan error, 1) + stream := &responseStream{events: []*sandboxexec.ExecuteCommandResponse{{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Start{Start: &sandboxexec.StartEvent{CommandId: new("wrong")}}}}}} + sandbox.ReadServerOutputForTest(stream, "expected", ready, stopped, io.Discard) + require.ErrorContains(t, <-stopped, "unexpected command ID") +} + +func TestReadServerOutputSuppressesDiagnosticsAfterReadiness(t *testing.T) { + stderr := func(text string) *sandboxexec.ExecuteCommandResponse { + return &sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stderr{Stderr: []byte(text)}}}}} + } + stream := &responseStream{events: []*sandboxexec.ExecuteCommandResponse{ + stderr("startup diagnostic\n"), + {ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stdout{Stdout: []byte("{\"port\":40000,\"token\":\"invocation-token\"}\n")}}}}}, + stderr("Info: Handover accepted\n"), + stderr("Error: context canceled\n"), + {ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_End{End: &sandboxexec.EndEvent{ExitCode: new(int32(0))}}}}, + }} + var diagnostics bytes.Buffer + ready := make(chan sandbox.Readiness, 1) + stopped := make(chan error, 1) + sandbox.ReadServerOutputForTest(stream, "command", ready, stopped, &diagnostics) + require.NoError(t, <-stopped) + assert.Equal(t, 40000, (<-ready).Port) + assert.Equal(t, "startup diagnostic\n", diagnostics.String()) +} + +func TestForwardingURL(t *testing.T) { + for _, tc := range []struct { + name string + dial proxy.DialRequest + query string + }{ + {"raw", proxy.DialRequest{ConnID: "connection"}, "id=connection"}, + {"raw-ignores-resume-fields", proxy.DialRequest{ConnID: "connection", Delivered: 4096, Reattach: true}, "id=connection"}, + {"initial", proxy.DialRequest{ConnID: "connection", ResumeCapable: true}, "delivered=0&id=connection&resume_version=2"}, + {"handover", proxy.DialRequest{ConnID: "connection", ResumeCapable: true, Delivered: 4096}, "delivered=4096&id=connection&resume_version=2"}, + {"reattach", proxy.DialRequest{ConnID: "connection", ResumeCapable: true, Delivered: 8192, Reattach: true}, "delivered=8192&id=connection&reattach=1&resume_version=2"}, + } { + t.Run(tc.name, func(t *testing.T) { + endpoint, err := sandbox.ForwardingURLForTest("https://workspace.test", "sandboxes/test", 40000, tc.dial) + require.NoError(t, err) + assert.Equal(t, "wss://workspace.test/sandbox-proxy/sandboxes/test/ports/40000/ssh?"+tc.query, endpoint) + }) + } + _, err := sandbox.ForwardingURLForTest("http://workspace.test", "sandboxes/test", 40000, proxy.DialRequest{ConnID: "connection"}) + require.Error(t, err) + _, err = sandbox.ForwardingURLForTest("https://workspace.test", "sandboxes/../other", 40000, proxy.DialRequest{ConnID: "connection"}) + require.Error(t, err) +} + +func TestPrepareRemoteRejectsPartialSetup(t *testing.T) { + home := t.TempDir() + ctx := env.WithUserHomeDir(t.Context(), home) + identity, err := sandbox.LocalIdentity(ctx, "https://workspace.test", "caller", "sandboxes/test") + require.NoError(t, err) + valid, _ := json.Marshal(map[string]string{"home": "/home/sandbox-agent", "config": "/home/sandbox-agent/config", "sshd": "/usr/sbin/sshd", "user": "sandbox-agent", "host_key": string(identity.PublicKey), "session_dir": "/home/sandbox-agent/session", "architecture": "x86_64"}) + for _, tc := range []struct { + name, status, stdout string + code int + truncated bool + errorText string + }{ + {name: "valid", status: "COMPLETED", stdout: string(valid)}, + {name: "failed", status: "FAILED", errorText: "setup failed"}, + {name: "exit", status: "COMPLETED", code: 2, errorText: "setup failed"}, + {name: "truncated", status: "COMPLETED", truncated: true, errorText: "setup failed"}, + {name: "invalid-json", status: "COMPLETED", stdout: "hello", errorText: "read sandbox SSH configuration"}, + {name: "unsupported-user", status: "COMPLETED", stdout: `{"user":"root"}`, errorText: "requires the sandbox-agent account"}, + {name: "incomplete", status: "COMPLETED", stdout: `{"user":"sandbox-agent"}`, errorText: "incomplete sandbox SSH configuration"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/sandbox-exec/sandboxes/test/exec-sync", r.URL.Path) + _ = json.NewEncoder(w).Encode(map[string]any{"status": "EXECUTE_COMMAND_STATUS_" + tc.status, "stdout": tc.stdout, "exit_code": tc.code, "truncated": tc.truncated}) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + _, err = sandbox.PrepareRemoteForTest(ctx, w, "sandboxes/test", identity) + if tc.errorText != "" { + require.ErrorContains(t, err, tc.errorText) + } else { + require.NoError(t, err) + contents, err := os.ReadFile(identity.KnownHostsPath) + require.NoError(t, err) + assert.Contains(t, string(contents), identity.HostKeyAlias) + } + }) + } +} + +// Closing an inherited blocking descriptor need not interrupt an in-flight read. +type blockedInput struct{ started, release, finished chan struct{} } + +func (input *blockedInput) Read([]byte) (int, error) { + close(input.started) + <-input.release + close(input.finished) + return 0, io.EOF +} +func (*blockedInput) Close() error { return nil } + +func TestProxyStreamStopsWithoutWaitingForBlockedStdin(t *testing.T) { + input := &blockedInput{started: make(chan struct{}), release: make(chan struct{}), finished: make(chan struct{})} + defer func() { close(input.release); <-input.finished }() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{} + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + _ = conn.WriteMessage(websocket.BinaryMessage, []byte("SSH-2.0-test\r\n")) + <-input.started + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + })) + defer server.Close() + finished := make(chan error, 1) + go func() { + finished <- sandbox.ProxyStreamForTest(t.Context(), input, io.Discard, false, func(ctx context.Context, _ proxy.DialRequest) (*websocket.Conn, error) { + conn, response, err := websocket.DefaultDialer.DialContext(ctx, strings.Replace(server.URL, "http://", "ws://", 1), nil) + if response != nil { + response.Body.Close() + } + return conn, err + }) + }() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("proxy waited for blocked inherited stdin after WebSocket closure") + } +} + +func TestReadServerOutputExitStatus(t *testing.T) { + for _, announced := range []bool{false, true} { + for _, tc := range []struct { + name string + code *int32 + }{ + {"success", new(int32(0))}, {"failure", new(int32(17))}, {"missing", nil}, + } { + t.Run(fmt.Sprintf("ready=%t/exit=%s", announced, tc.name), func(t *testing.T) { + stream := &responseStream{} + if announced { + stream.events = append(stream.events, &sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stdout{Stdout: []byte("{\"port\":40000,\"token\":\"test-token\"}\n")}}}}}) + } + stream.events = append(stream.events, &sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_End{End: &sandboxexec.EndEvent{ExitCode: tc.code}}}}) + ready := make(chan sandbox.Readiness, 1) + stopped := make(chan error, 1) + sandbox.ReadServerOutputForTest(stream, "command", ready, stopped, io.Discard) + err := <-stopped + if announced && tc.name == "success" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } + } +} + +type untouchedInput struct{ reads, closes atomic.Int32 } + +func (input *untouchedInput) Read([]byte) (int, error) { input.reads.Add(1); return 0, io.EOF } +func (input *untouchedInput) Close() error { input.closes.Add(1); return nil } + +func TestProxyStreamPreservesInputWhenUpgradeFails(t *testing.T) { + input := new(untouchedInput) + err := sandbox.ProxyStreamForTest(t.Context(), input, io.Discard, true, func(context.Context, proxy.DialRequest) (*websocket.Conn, error) { + return nil, errors.New("upgrade failed") + }) + require.ErrorIs(t, err, proxy.ErrConnectFailed) + assert.Zero(t, input.reads.Load()) + assert.Zero(t, input.closes.Load()) +} diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go index 9f6cab4f9fd..d274ccc73dd 100644 --- a/experimental/ssh/sandbox/server.go +++ b/experimental/ssh/sandbox/server.go @@ -23,9 +23,10 @@ const RelayTokenHeader = "X-Sandbox-SSH-Token" // ServerOptions describes a relay whose SSH configuration was prepared by the caller. type ServerOptions struct { - SSHD string - Config string - ConnectTimeout time.Duration + SSHD string + Config string + ConnectTimeout time.Duration + OutputHeartbeat time.Duration } // Readiness is written as one JSON line after the sshd configuration is validated. @@ -46,6 +47,14 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu if opts.Config == "" || opts.SSHD == "" { return errors.New("sshd executable and configuration are required") } + var heartbeatOutput io.WriteCloser + if opts.OutputHeartbeat > 0 { + var ok bool + heartbeatOutput, ok = output.(io.WriteCloser) + if !ok { + return errors.New("command heartbeat output must support closing") + } + } check := exec.CommandContext(ctx, opts.SSHD, "-t", "-f", opts.Config) if diagnostic, err := check.CombinedOutput(); err != nil { return fmt.Errorf("validate sshd configuration: %w: %s", err, diagnostic) @@ -101,12 +110,40 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu if err := json.NewEncoder(output).Encode(Readiness{Port: listener.Addr().(*net.TCPAddr).Port, Token: token, ResumeVersion: proxy.ResumeProtocolVersion}); err != nil { return fmt.Errorf("report relay readiness: %w", err) } + heartbeatErrors := make(chan error, 1) + if opts.OutputHeartbeat > 0 { + heartbeatDone := make(chan struct{}) + defer func() { + cancel() + heartbeatOutput.Close() + <-heartbeatDone + }() + go func() { + defer close(heartbeatDone) + ticker := time.NewTicker(opts.OutputHeartbeat) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + // SSH traffic travels over a separate WebSocket connection. + if _, err := io.WriteString(heartbeatOutput, "\n"); err != nil { + heartbeatErrors <- err + return + } + } + } + }() + } log.Infof(ctx, "Sandbox SSH server ready on port %d", listener.Addr().(*net.TCPAddr).Port) timer := time.NewTimer(opts.ConnectTimeout) defer timer.Stop() firstConnection := started for { select { + case err := <-heartbeatErrors: + return fmt.Errorf("write command heartbeat: %w", err) case <-ctx.Done(): return ctx.Err() case <-firstConnection: diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index dbad9a0317f..bbaac183b12 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -31,6 +31,20 @@ import ( ) func TestMain(m *testing.M) { + if endpoint := os.Getenv("DATABRICKS_TEST_SANDBOX_PROXY_URL"); endpoint != "" { + // TestMain also runs as a standalone SSH transport subprocess. + //nolint:gocritic // TestMain has no testing.T context. + err := sandbox.ProxyStreamForTest(context.Background(), os.Stdin, os.Stdout, true, func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + conn, err := dialWebSocket(ctx, resumeEndpoint(endpoint, request)) + return conn, err + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + } + if os.Getenv("DATABRICKS_TEST_SANDBOX_SSHD") == "1" { if os.Args[1] == "-t" { os.Exit(0) @@ -109,6 +123,38 @@ func TestServerIdleShutdown(t *testing.T) { } } +func TestCommandHeartbeatDoesNotPreventIdleShutdown(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader, writer := io.Pipe() + defer reader.Close() + t.Cleanup(func() { writer.Close() }) + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: executable, Config: "test-config", ConnectTimeout: time.Second, OutputHeartbeat: 10 * time.Millisecond}, writer) + }() + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + var newline [1]byte + _, err = io.ReadFull(reader, newline[:]) + require.NoError(t, err) + assert.Equal(t, byte('\n'), newline[0]) + // Stop draining output: the next heartbeat blocks, but idle shutdown must not. + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("stalled command output prevented idle shutdown") + } + _, err = writer.Write([]byte("after shutdown")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} + func TestServerRejectsInvalidConfigurationBeforeReadiness(t *testing.T) { listener, err := net.Listen("tcp4", "127.0.0.1:0") require.NoError(t, err) @@ -346,6 +392,51 @@ type blockedReadiness struct { release chan struct{} } +type failingHeartbeat struct { + ready bool + err error +} + +func (output *failingHeartbeat) Write(data []byte) (int, error) { + if !output.ready { + output.ready = true + return len(data), nil + } + return 0, output.err +} + +func (*failingHeartbeat) Close() error { return nil } + +func TestServerStopsWhenCommandHeartbeatFails(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + brokenPipe := errors.New("command output pipe closed") + output := &failingHeartbeat{err: brokenPipe} + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{ + SSHD: executable, Config: "test-config", ConnectTimeout: time.Minute, OutputHeartbeat: 10 * time.Millisecond, + }, output) + }() + select { + case err := <-finished: + require.ErrorIs(t, err, brokenPipe) + require.ErrorContains(t, err, "write command heartbeat") + case <-time.After(5 * time.Second): + t.Fatal("server ignored its broken command-output pipe") + } + connection, err := net.DialTimeout("tcp4", listener.Addr().String(), time.Second) + if connection != nil { + connection.Close() + } + require.Error(t, err, "server listener must close after the heartbeat fails") +} + func (writer blockedReadiness) Write(data []byte) (int, error) { close(writer.entered) <-writer.release diff --git a/experimental/ssh/sandbox/setup.py b/experimental/ssh/sandbox/setup.py new file mode 100644 index 00000000000..a0c20fad33c --- /dev/null +++ b/experimental/ssh/sandbox/setup.py @@ -0,0 +1,121 @@ +import fcntl +import hashlib +import json +import os +import pwd +import shutil +import subprocess +import sys +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + + +def atomic_write(path, content): + fd, temporary = tempfile.mkstemp(dir=path.parent) + try: + with os.fdopen(fd, "w") as output: + output.write(content) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +request = json.loads(sys.argv[1]) +sshd = shutil.which("sshd") or "/usr/sbin/sshd" +if not os.path.isfile(sshd): + raise RuntimeError("Install openssh-server in the sandbox before connecting") +user = pwd.getpwuid(os.getuid()).pw_name +home = Path.home() +if home.stat().st_mode & 0o022: + raise RuntimeError("SSH requires HOME not to be writable by group or other users") +parent = home / ".databricks" +parent.mkdir(mode=0o700, exist_ok=True) +# StrictModes checks every directory above authorized_keys, including existing ones. +parent.chmod(parent.stat().st_mode & ~0o022) +base = home / ".databricks" / "sandbox-ssh" +base.mkdir(mode=0o700, exist_ok=True) +base.chmod(0o700) +key = request["public_key"].strip() +if len(key.split()) != 2 or not key.startswith("ssh-ed25519 "): + raise ValueError("Expected one Ed25519 public key") +key_dir = base / hashlib.sha256(key.encode()).hexdigest() +key_dir.mkdir(mode=0o700, exist_ok=True) +key_dir.chmod(0o700) +# Each client key has its own file, so renewal cannot discard another caller's entry. +authorized = key_dir / "authorized_keys" +# The server runs with TZ=UTC. Older OpenSSH releases reject a trailing Z. +expiry = (datetime.now(timezone.utc) + timedelta(hours=24)).strftime("%Y%m%d%H%M%S") +atomic_write(authorized, 'expiry-time="' + expiry + '" ' + key + "\n") +# Server keys belong to this persistent HOME, and are never returned to the client. +# Serialize generation so simultaneous connections share the same server identity. +with open(base / "host-key.lock", "a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + host_key = base / "host_ed25519" + if not host_key.exists(): + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(host_key)], check=True) + host_public = subprocess.check_output(["ssh-keygen", "-y", "-f", str(host_key)], universal_newlines=True).strip() +# The server inherits this lock across exec; a crashed client cannot make an +# active session eligible for deletion, even after a day. +# Serialize pruners so none recreates server.lock while another removes it. +with open(base / "sessions.lock", "a") as cleanup_lock: + fcntl.flock(cleanup_lock, fcntl.LOCK_EX) + for old_session in base.glob("session-*"): + try: + if ( + old_session.is_symlink() + or not old_session.is_dir() + or old_session.stat().st_mtime >= time.time() - 86400 + ): + continue + with open(old_session / "server.lock", "a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + shutil.rmtree(old_session) + except (BlockingIOError, FileNotFoundError): + pass +session = Path(tempfile.mkdtemp(prefix="session-", dir=base)) +config = session / "sshd_config" + + +def quote(value): + value = str(value) + if any(c in value for c in ('"', "\n", "\r", "\\")): + raise ValueError("Unsupported character in SSH configuration path") + return '"' + value + '"' + + +atomic_write( + config, + "\n".join( + [ + "HostKey " + quote(host_key), + "AuthorizedKeysFile " + quote(authorized), + "PubkeyAuthentication yes", + "PasswordAuthentication no", + "KbdInteractiveAuthentication no", + # OpenSSH requires root for PAM; this server runs as sandbox-agent. + "UsePAM no", + "AllowUsers " + user, + "AllowTcpForwarding yes", + "ClientAliveInterval 30", + "Subsystem sftp internal-sftp", + "PidFile " + quote(session / "sshd.pid"), + ] + ) + + "\n", +) +print( + json.dumps( + { + "home": str(home), + "config": str(config), + "sshd": sshd, + "user": user, + "host_key": host_public, + "architecture": os.uname().machine, + "session_dir": str(session), + } + ) +) diff --git a/go.mod b/go.mod index 1c6562f5ad4..240449575cd 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,8 @@ require ( golang.org/x/sys v0.49.0 // BSD-3-Clause golang.org/x/term v0.47.0 // BSD-3-Clause golang.org/x/text v0.43.0 // BSD-3-Clause + google.golang.org/grpc v1.83.2 // Apache-2.0 + google.golang.org/protobuf v1.36.11 // BSD-3-Clause gopkg.in/ini.v1 v1.67.3 // Apache-2.0 ) @@ -97,7 +99,5 @@ require ( golang.org/x/time v0.14.0 // indirect google.golang.org/api v0.265.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.83.2 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/build/notice_test.go b/internal/build/notice_test.go index abe1d39287d..bbc9651832c 100644 --- a/internal/build/notice_test.go +++ b/internal/build/notice_test.go @@ -14,9 +14,11 @@ import ( // moduleToGitHub maps non-GitHub go.mod paths to their GitHub org/repo slug. var moduleToGitHub = map[string]string{ - "gopkg.in/ini.v1": "go-ini/ini", - "go.yaml.in/yaml/v3": "yaml/go-yaml", - "dario.cat/mergo": "darccio/mergo", + "google.golang.org/grpc": "grpc/grpc-go", + "google.golang.org/protobuf": "protocolbuffers/protobuf-go", + "gopkg.in/ini.v1": "go-ini/ini", + "go.yaml.in/yaml/v3": "yaml/go-yaml", + "dario.cat/mergo": "darccio/mergo", } // Modules excluded from NOTICE requirements (Databricks-owned).