From 5ce87693a13f0d14c392c2150ead69c286021870 Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Wed, 18 Nov 2026 05:30:42 +0000 Subject: [PATCH 1/7] Add internal sandbox WebSocket SSH server --- .github/CODEOWNERS | 1 + acceptance/cmd/sandbox/server/out.test.toml | 3 + acceptance/cmd/sandbox/server/output.txt | 5 + acceptance/cmd/sandbox/server/script | 1 + acceptance/cmd/sandbox/server/server_test.py | 71 ++++ acceptance/cmd/sandbox/server/test.toml | 4 + cmd/cmd.go | 12 +- .../ssh/internal/proxy/client_server_test.go | 30 ++ .../ssh/internal/proxy/connections.go | 43 +- .../ssh/internal/proxy/connections_test.go | 13 + experimental/ssh/internal/proxy/server.go | 67 +++- .../ssh/internal/proxy/server_options_test.go | 84 ++++ experimental/ssh/sandbox/command.go | 55 +++ experimental/ssh/sandbox/command_test.go | 67 ++++ experimental/ssh/sandbox/server.go | 157 ++++++++ experimental/ssh/sandbox/server_test.go | 370 ++++++++++++++++++ 16 files changed, 969 insertions(+), 14 deletions(-) create mode 100644 acceptance/cmd/sandbox/server/out.test.toml create mode 100644 acceptance/cmd/sandbox/server/output.txt create mode 100644 acceptance/cmd/sandbox/server/script create mode 100644 acceptance/cmd/sandbox/server/server_test.py create mode 100644 acceptance/cmd/sandbox/server/test.toml create mode 100644 experimental/ssh/internal/proxy/server_options_test.go create mode 100644 experimental/ssh/sandbox/command.go create mode 100644 experimental/ssh/sandbox/command_test.go create mode 100644 experimental/ssh/sandbox/server.go create mode 100644 experimental/ssh/sandbox/server_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4b1a9df7bb4..f46faa9b899 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -26,6 +26,7 @@ # SSH tunnel /experimental/ssh/ @databricks/eng-deco-cli @databricks/eng-deco-ax +/experimental/ssh/sandbox/ @databricks/eng-deco-cli @databricks/eng-sandbox /acceptance/ssh/ @databricks/eng-deco-cli @databricks/eng-deco-ax # Apps diff --git a/acceptance/cmd/sandbox/server/out.test.toml b/acceptance/cmd/sandbox/server/out.test.toml new file mode 100644 index 00000000000..0f8449364f7 --- /dev/null +++ b/acceptance/cmd/sandbox/server/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +GOOS.darwin = false +GOOS.windows = false diff --git a/acceptance/cmd/sandbox/server/output.txt b/acceptance/cmd/sandbox/server/output.txt new file mode 100644 index 00000000000..8f4500bf407 --- /dev/null +++ b/acceptance/cmd/sandbox/server/output.txt @@ -0,0 +1,5 @@ + +>>> python3 server_test.py +Server remains hidden from sandbox help +Server starts without workspace credentials +SIGTERM stops the server and its SSH child diff --git a/acceptance/cmd/sandbox/server/script b/acceptance/cmd/sandbox/server/script new file mode 100644 index 00000000000..e40a2695bc0 --- /dev/null +++ b/acceptance/cmd/sandbox/server/script @@ -0,0 +1 @@ +trace python3 server_test.py diff --git a/acceptance/cmd/sandbox/server/server_test.py b/acceptance/cmd/sandbox/server/server_test.py new file mode 100644 index 00000000000..1d4589d4428 --- /dev/null +++ b/acceptance/cmd/sandbox/server/server_test.py @@ -0,0 +1,71 @@ +import base64 +import json +import os +import re +import signal +import socket +import subprocess +import tempfile +import time +from pathlib import Path + +cli = os.environ["CLI"] +environment = { + key: value + for key, value in os.environ.items() + if not key.startswith(("DATABRICKS_", "ARM_", "AZURE_", "AWS_", "GOOGLE_")) +} +with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + environment["HOME"] = temporary + environment["DATABRICKS_CONFIG_FILE"] = str(directory / "missing-config") + child_pid = directory / "child.pid" + sshd = directory / "sshd" + # A long-running child exercises cancellation without requiring system sshd. + sshd.write_text('#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\necho $$ > "' + str(child_pid) + '"\nexec cat\n') + sshd.chmod(0o700) + help_result = subprocess.run( + [cli, "sandbox", "--help"], env=environment, capture_output=True, text=True, check=True + ) + assert not re.search(r"^\s+server\s", help_result.stdout, re.MULTILINE) + print("Server remains hidden from sandbox help") + process = subprocess.Popen( + [cli, "sandbox", "server", "--port", "0", "--sshd", str(sshd), "--sshd-config", "prepared-config"], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + ready = json.loads(process.stdout.readline()) + assert 10000 <= ready["port"] <= 65535 + assert ready["token"] + print("Server starts without workspace credentials") + with socket.create_connection(("127.0.0.1", ready["port"]), timeout=5) as connection: + key = base64.b64encode(os.urandom(16)).decode() + connection.sendall( + ( + "GET /ssh?id=acceptance HTTP/1.1\r\nHost: localhost\r\n" + "Upgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\n" + f"Sec-WebSocket-Key: {key}\r\nX-Sandbox-SSH-Token: {ready['token']}\r\n\r\n" + ).encode() + ) + assert connection.recv(4096).startswith(b"HTTP/1.1 101") + deadline = time.monotonic() + 5 + while not child_pid.exists(): + assert time.monotonic() < deadline, "sshd child did not start" + time.sleep(0.01) + pid = int(child_pid.read_text()) + process.send_signal(signal.SIGTERM) + assert process.wait(timeout=10) == 1 # context cancellation, rather than signal termination + try: + os.kill(pid, 0) + except ProcessLookupError: + pass + else: + raise AssertionError("sshd child survived server shutdown") + print("SIGTERM stops the server and its SSH child") + finally: + if process.poll() is None: + process.kill() + process.wait() diff --git a/acceptance/cmd/sandbox/server/test.toml b/acceptance/cmd/sandbox/server/test.toml new file mode 100644 index 00000000000..e0829cb064d --- /dev/null +++ b/acceptance/cmd/sandbox/server/test.toml @@ -0,0 +1,4 @@ +# The server and its signal handling run inside Linux sandboxes. +GOOS.windows = false +GOOS.darwin = false +Ignore = [".databricks", ".ssh", "empty-home", "child.pid"] diff --git a/cmd/cmd.go b/cmd/cmd.go index 538a2fb9a26..e1cf2fdabef 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -8,6 +8,7 @@ import ( aitoolscmd "github.com/databricks/cli/cmd/aitools" "github.com/databricks/cli/cmd/psql" ssh "github.com/databricks/cli/experimental/ssh/cmd" + sandboxssh "github.com/databricks/cli/experimental/ssh/sandbox" "github.com/databricks/cli/cmd/account" "github.com/databricks/cli/cmd/api" @@ -91,6 +92,15 @@ func New(ctx context.Context) *root.Tree { return cli } +func sandboxSpec() root.CommandSpec { + spec := sandbox.Spec + spec.Build = func(ctx context.Context, cmd *cobra.Command) { + sandbox.Spec.Build(ctx, cmd) + cmd.AddCommand(sandboxssh.NewServerCommand()) + } + return spec +} + // topLevelSpecs returns the specs of the CLI's top-level commands. func topLevelSpecs() []root.CommandSpec { specs := []root.CommandSpec{ @@ -108,7 +118,7 @@ func topLevelSpecs() []root.CommandSpec { configure.Spec, fs.Spec, labs.Spec, - sandbox.Spec, + sandboxSpec(), sync.Spec, version.Spec, quickstart.Spec, diff --git a/experimental/ssh/internal/proxy/client_server_test.go b/experimental/ssh/internal/proxy/client_server_test.go index d215612c151..6ed84ddbce2 100644 --- a/experimental/ssh/internal/proxy/client_server_test.go +++ b/experimental/ssh/internal/proxy/client_server_test.go @@ -562,3 +562,33 @@ func TestClientTimesOutWhenServerSendsNothing(t *testing.T) { t.Fatal("RunClientProxy did not abort on the handshake timeout") } } + +func TestSingleSessionCannotStartAgainAfterDisconnect(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + defer manager.Close() + var started atomic.Int32 + handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { + started.Add(1) + return exec.CommandContext(ctx, "cat") + }, WithSingleSession()) + server := httptest.NewServer(handler) + defer server.Close() + connection, response, err := websocket.DefaultDialer.Dial("ws"+server.URL[4:]+"?id=session", nil) + if response != nil { + response.Body.Close() + } + require.NoError(t, err) + require.NoError(t, connection.WriteMessage(websocket.BinaryMessage, []byte("first-session"))) + require.NoError(t, connection.SetReadDeadline(time.Now().Add(5*time.Second))) + _, message, err := connection.ReadMessage() + require.NoError(t, err) + require.Equal(t, "first-session", string(message)) + require.NoError(t, connection.Close()) + require.Eventually(t, func() bool { return manager.Count() == 0 }, 5*time.Second, time.Millisecond) + for _, id := range []string{"session", "another-session"} { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id="+id, nil)) + require.Equal(t, http.StatusServiceUnavailable, response.Code) + } + require.Equal(t, int32(1), started.Load()) +} diff --git a/experimental/ssh/internal/proxy/connections.go b/experimental/ssh/internal/proxy/connections.go index 3e3991571dc..b025528a071 100644 --- a/experimental/ssh/internal/proxy/connections.go +++ b/experimental/ssh/internal/proxy/connections.go @@ -8,21 +8,33 @@ import ( // ConnectionsManager manages concurrent websocket clients and sends a shutdown signal if no // clients are connected for a specified duration. type ConnectionsManager struct { - maxClients int - shutdownDelay time.Duration - shutdownTimer *time.Timer - shutdownTimerMu sync.Mutex - connections map[string]*proxyConnection - connectionsMu sync.Mutex - TimedOut chan bool + maxClients int + shutdownDelay time.Duration + shutdownTimer *time.Timer + shutdownTimerMu sync.Mutex + disableIdleShutdown bool + connections map[string]*proxyConnection + connectionsMu sync.Mutex + TimedOut chan bool } -func NewConnectionsManager(maxClients int, shutdownDelay time.Duration) *ConnectionsManager { +// ConnectionsManagerOption configures connection lifetime management. +type ConnectionsManagerOption func(*ConnectionsManager) + +// WithoutIdleShutdown leaves server lifetime management to the caller. +func WithoutIdleShutdown() ConnectionsManagerOption { + return func(cm *ConnectionsManager) { cm.disableIdleShutdown = true } +} + +func NewConnectionsManager(maxClients int, shutdownDelay time.Duration, options ...ConnectionsManagerOption) *ConnectionsManager { cm := &ConnectionsManager{ maxClients: maxClients, shutdownDelay: shutdownDelay, connections: make(map[string]*proxyConnection), - TimedOut: make(chan bool), + TimedOut: make(chan bool, 1), + } + for _, option := range options { + option(cm) } cm.startShutdownTimer(shutdownDelay) return cm @@ -80,13 +92,19 @@ func (cm *ConnectionsManager) ExtendIdleTimeout(delay time.Duration) { } func (cm *ConnectionsManager) startShutdownTimer(delay time.Duration) { + if cm.disableIdleShutdown { + return + } cm.shutdownTimerMu.Lock() defer cm.shutdownTimerMu.Unlock() if cm.shutdownTimer != nil { cm.shutdownTimer.Stop() } cm.shutdownTimer = time.AfterFunc(delay, func() { - cm.TimedOut <- true + select { + case cm.TimedOut <- true: + default: + } }) } @@ -97,3 +115,8 @@ func (cm *ConnectionsManager) cancelShutdownTimer() { cm.shutdownTimer.Stop() } } + +// Close stops the idle timer. +func (cm *ConnectionsManager) Close() { + cm.cancelShutdownTimer() +} diff --git a/experimental/ssh/internal/proxy/connections_test.go b/experimental/ssh/internal/proxy/connections_test.go index e05c92d1fcb..a55ed3ebe1c 100644 --- a/experimental/ssh/internal/proxy/connections_test.go +++ b/experimental/ssh/internal/proxy/connections_test.go @@ -239,3 +239,16 @@ func TestConnectionsManager_ShutdownTimer_NoRestartWhenConnectionsRemain(t *test assert.False(t, timedOut, "Unexpected timeout signal while connections still exist") }) } + +func TestConnectionsManagerCloseStopsIdleTimer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cm := NewConnectionsManager(1, time.Second) + cm.Close() + time.Sleep(2 * time.Second) + select { + case <-cm.TimedOut: + t.Fatal("closed manager timed out") + default: + } + }) +} diff --git a/experimental/ssh/internal/proxy/server.go b/experimental/ssh/internal/proxy/server.go index fd5a93417ba..dde33c00d5d 100644 --- a/experimental/ssh/internal/proxy/server.go +++ b/experimental/ssh/internal/proxy/server.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strconv" + "sync" "time" "github.com/databricks/cli/libs/log" @@ -23,15 +24,44 @@ type proxyServer struct { connections *ConnectionsManager createServerCommand createServerCommandFunc resumeGrace time.Duration + allowResume bool + allowHandover bool + singleSession bool + sessionUsed bool + sessionMu sync.Mutex } -func NewProxyServer(ctx context.Context, connections *ConnectionsManager, createServerCommand createServerCommandFunc) *proxyServer { - return &proxyServer{ +// ServerOption configures the SSH transport capabilities. +type ServerOption func(*proxyServer) + +// WithoutSessionResume rejects requests to resume an interrupted session. +func WithoutSessionResume() ServerOption { + return func(server *proxyServer) { server.allowResume = false } +} + +// WithoutHandover rejects requests to switch an active session to another WebSocket. +func WithoutHandover() ServerOption { + return func(server *proxyServer) { server.allowHandover = false } +} + +// WithSingleSession permits handovers but never starts a second SSH session. +func WithSingleSession() ServerOption { + return func(server *proxyServer) { server.singleSession = true } +} + +func NewProxyServer(ctx context.Context, connections *ConnectionsManager, createServerCommand createServerCommandFunc, options ...ServerOption) *proxyServer { + server := &proxyServer{ ctx: ctx, connections: connections, createServerCommand: createServerCommand, resumeGrace: proxyResumeGrace, + allowResume: true, + allowHandover: true, } + for _, option := range options { + option(server) + } + return server } func (server *proxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -45,12 +75,20 @@ func (server *proxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + if !server.allowResume && (req.ResumeCapable || req.Reattach) { + http.Error(w, "Session resume is not supported", http.StatusBadRequest) + return + } ctx := log.NewContext(server.ctx, log.GetLogger(server.ctx).With("session", id)) conn, exists := server.connections.Get(id) switch { case exists && conn != nil && req.Reattach: server.handleReattach(ctx, w, r, conn, req) case exists && conn != nil: + if !server.allowHandover { + http.Error(w, "Session handover is not supported", http.StatusBadRequest) + return + } server.handleExistingConnection(ctx, w, r, conn) case req.Reattach: // The session is gone: its grace period expired, or the server restarted. Say so instead @@ -117,6 +155,26 @@ func (server *proxyServer) handleExistingConnection(ctx context.Context, w http. } func (server *proxyServer) handleNewConnection(ctx context.Context, w http.ResponseWriter, r *http.Request, id string, req DialRequest) { + // Failed upgrades can retry; a started SSH session consumes this server permanently. + accepted := false + if server.singleSession { + server.sessionMu.Lock() + if server.sessionUsed { + server.sessionMu.Unlock() + http.Error(w, "SSH session already used", http.StatusServiceUnavailable) + return + } + server.sessionUsed = true + server.sessionMu.Unlock() + defer func() { + if !accepted { + server.sessionMu.Lock() + server.sessionUsed = false + server.sessionMu.Unlock() + } + }() + } + // The server never dials, so it passes no connection factory: reattaching, for it, means // waiting for the client to come back. var conn *proxyConnection @@ -134,7 +192,10 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo defer server.connections.Remove(id) log.Infof(ctx, "Starting proxy server for new connection, count: %d", server.connections.Count()) - err := runServerProxy(ctx, conn, server.createServerCommand, w, r) + err := runServerProxy(ctx, conn, func(ctx context.Context) *exec.Cmd { + accepted = true + return server.createServerCommand(ctx) + }, w, r) if err != nil { log.Errorf(ctx, "Proxy server error: %v", err) } else { diff --git a/experimental/ssh/internal/proxy/server_options_test.go b/experimental/ssh/internal/proxy/server_options_test.go new file mode 100644 index 00000000000..99a98ad535c --- /dev/null +++ b/experimental/ssh/internal/proxy/server_options_test.go @@ -0,0 +1,84 @@ +package proxy + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestServerCanDisableResumeAndHandover(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + defer manager.Close() + handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { + t.Error("rejected requests must not start an SSH process") + return exec.CommandContext(ctx, "unused") + }, WithoutSessionResume(), WithoutHandover()) + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/ssh?id=test&delivered=0&resume_version=%d", ResumeProtocolVersion), nil)) + require.Equal(t, http.StatusBadRequest, response.Code) + require.Contains(t, response.Body.String(), "resume is not supported") + require.True(t, manager.TryAdd("test", newProxyConnection(nil))) + response = httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id=test", nil)) + require.Equal(t, http.StatusBadRequest, response.Code) + require.Contains(t, response.Body.String(), "handover is not supported") +} + +func TestManagerCanDisableIdleShutdown(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + defer manager.Close() + require.True(t, manager.TryAdd("session", newProxyConnection(nil))) + manager.Remove("session") + select { + case <-manager.TimedOut: + t.Fatal("idle shutdown must stay disabled") + case <-time.After(20 * time.Millisecond): + } +} + +func TestSingleSessionRejectsReuseAfterManagerRemoval(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + defer manager.Close() + handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { + t.Fatal("a completed logical session must never start another process") + return nil + }, WithSingleSession()) + // Reproduce the gap after Remove and before the outer sandbox handler returns. + handler.sessionUsed = true + for _, id := range []string{"same-session", "different-session"} { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id="+id, nil)) + require.Equal(t, http.StatusServiceUnavailable, response.Code) + } +} + +func TestSingleSessionFailedUpgradeDoesNotConsumeSession(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + defer manager.Close() + handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { + t.Fatal("a non-WebSocket request must never start a process") + return nil + }, WithSingleSession()) + for range 2 { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id=test", nil)) + require.Equal(t, http.StatusBadRequest, response.Code) + require.False(t, handler.sessionUsed) + } +} + +func TestZeroIdleTimeoutStillSignalsImmediately(t *testing.T) { + manager := NewConnectionsManager(1, 0) + defer manager.Close() + select { + case <-manager.TimedOut: + case <-time.After(5 * time.Second): + t.Fatal("zero duration must preserve compute's immediate idle shutdown") + } +} diff --git a/experimental/ssh/sandbox/command.go b/experimental/ssh/sandbox/command.go new file mode 100644 index 00000000000..f25c0c67d11 --- /dev/null +++ b/experimental/ssh/sandbox/command.go @@ -0,0 +1,55 @@ +package sandbox + +import ( + "errors" + "fmt" + "net" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" +) + +// NewServerCommand returns the internal sandbox SSH server command. +func NewServerCommand() *cobra.Command { + var config, sshd string + var port int + var connectTimeout time.Duration + cmd := &cobra.Command{ + Use: "server", Short: "Run the sandbox WebSocket SSH relay", Hidden: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if port != 0 && !validForwardPort(port) { + return errors.New("port must be zero or between 10000 and 65535, excluding 10101, 18767, and 19090") + } + if connectTimeout <= 0 { + return errors.New("connection timeout must be positive") + } + listener, err := net.Listen("tcp4", fmt.Sprintf("0.0.0.0:%d", port)) + if err != nil { + return fmt.Errorf("listen for SSH WebSockets: %w", err) + } + if !validForwardPort(listener.Addr().(*net.TCPAddr).Port) { + listener.Close() + return errors.New("selected listener port is not supported by sandbox forwarding; retry the command") + } + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGTERM, os.Interrupt) + defer stop() + return Serve(ctx, listener, ServerOptions{ + SSHD: sshd, Config: config, ConnectTimeout: connectTimeout, + }, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&config, "sshd-config", "", "Prepared sshd configuration file") + cmd.MarkFlagRequired("sshd-config") + cmd.Flags().StringVar(&sshd, "sshd", "/usr/sbin/sshd", "OpenSSH server executable") + cmd.Flags().IntVar(&port, "port", 40000, "HTTP/WebSocket listener port (0 selects a free port)") + cmd.Flags().DurationVar(&connectTimeout, "connect-timeout", 2*time.Minute, "Stop if no SSH connection arrives within this duration after readiness") + return cmd +} + +func validForwardPort(port int) bool { + return port >= 10000 && port <= 65535 && port != 10101 && port != 18767 && port != 19090 +} diff --git a/experimental/ssh/sandbox/command_test.go b/experimental/ssh/sandbox/command_test.go new file mode 100644 index 00000000000..2690d8953ee --- /dev/null +++ b/experimental/ssh/sandbox/command_test.go @@ -0,0 +1,67 @@ +package sandbox_test + +import ( + "bytes" + "encoding/json" + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/databricks/cli/experimental/ssh/sandbox" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServerCommandValidatesArguments(t *testing.T) { + for _, args := range [][]string{ + {"--sshd-config", "config", "--port", "22"}, + {"--sshd-config", "config", "--connect-timeout", "0s"}, + {"--sshd-config", "config", "unexpected"}, + } { + cmd := sandbox.NewServerCommand() + cmd.SetArgs(args) + err := cmd.ExecuteContext(t.Context()) + require.Error(t, err) + } +} + +func TestServerCommandReportsReadiness(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sandbox servers run on Linux") + } + binary := filepath.Join(t.TempDir(), "sshd") + require.NoError(t, os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o700)) + cmd := sandbox.NewServerCommand() + var output bytes.Buffer + cmd.SetOut(&output) + cmd.SetArgs([]string{"--sshd", binary, "--sshd-config", "prepared-config", "--port", "0", "--connect-timeout", "20ms"}) + require.NoError(t, cmd.ExecuteContext(t.Context())) + var ready struct { + Port int `json:"port"` + } + require.NoError(t, json.Unmarshal(output.Bytes(), &ready)) + assert.GreaterOrEqual(t, ready.Port, 10000) +} + +func TestServerCommandReportsOccupiedPort(t *testing.T) { + listener, err := net.Listen("tcp4", "0.0.0.0:0") + require.NoError(t, err) + defer listener.Close() + cmd := sandbox.NewServerCommand() + cmd.SetArgs([]string{"--sshd-config", "prepared-config", "--port", strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "listen for SSH WebSockets") +} + +func TestServerCommandRejectsNonForwardablePorts(t *testing.T) { + for _, port := range []string{"9999", "10101", "18767", "19090", "65536"} { + t.Run(port, func(t *testing.T) { + cmd := sandbox.NewServerCommand() + cmd.SetArgs([]string{"--sshd-config", "config", "--port", port}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "port must be zero") + }) + } +} diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go new file mode 100644 index 00000000000..0a50eb0395f --- /dev/null +++ b/experimental/ssh/sandbox/server.go @@ -0,0 +1,157 @@ +package sandbox + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os/exec" + "sync" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/libs/log" +) + +// RelayTokenHeader authenticates connections to this invocation's relay. +const RelayTokenHeader = "X-Sandbox-SSH-Token" + +// ServerOptions describes a relay whose SSH configuration was prepared by the caller. +type ServerOptions struct { + SSHD string + Config string + ConnectTimeout time.Duration +} + +// Readiness is written as one JSON line after the sshd configuration is validated. +// Token is only returned over the caller's authenticated command-output stream. +type Readiness struct { + Port int `json:"port"` + Token string `json:"token"` +} + +// Serve relays /ssh WebSockets to one sshd session, including its handovers. +// It owns listener and stops its SSH subprocess before returning. +func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, output io.Writer) error { + defer listener.Close() + if opts.ConnectTimeout <= 0 { + return errors.New("connection timeout must be positive") + } + if opts.Config == "" || opts.SSHD == "" { + return errors.New("sshd executable and configuration are required") + } + check := exec.CommandContext(ctx, opts.SSHD, "-t", "-f", opts.Config) + if diagnostic, err := check.CombinedOutput(); err != nil { + return fmt.Errorf("validate sshd configuration: %w: %s", err, diagnostic) + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + connections := proxy.NewConnectionsManager(1, 0, proxy.WithoutIdleShutdown()) + defer connections.Close() + token := rand.Text() + started, completed := make(chan struct{}), make(chan struct{}) + var requests sync.WaitGroup + var mu sync.Mutex + var sessionID string + admitting, closing := false, false + handler := proxy.NewProxyServer(ctx, connections, func(ctx context.Context) *exec.Cmd { + mu.Lock() + admitting = false + close(started) + mu.Unlock() + return exec.CommandContext(ctx, opts.SSHD, "-i", "-f", opts.Config) + }, proxy.WithoutSessionResume(), proxy.WithSingleSession()) + mux := http.NewServeMux() + mux.HandleFunc("GET /ssh", func(w http.ResponseWriter, r *http.Request) { + if subtle.ConstantTimeCompare([]byte(r.Header.Get(RelayTokenHeader)), []byte(token)) != 1 { + log.Info(ctx, "Rejecting sandbox SSH connection without its invocation token") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + id := r.URL.Query().Get("id") + if id == "" { + http.Error(w, "Missing 'id' query parameter", http.StatusBadRequest) + return + } + mu.Lock() + if closing || admitting || (sessionID != "" && sessionID != id) { + mu.Unlock() + log.Info(ctx, "Rejecting another sandbox SSH session") + http.Error(w, "SSH session already active or server is stopping", http.StatusServiceUnavailable) + return + } + first := sessionID == "" + if first { + sessionID, admitting = id, true + } + requests.Add(1) + mu.Unlock() + defer func() { + mu.Lock() + if first { + if admitting { + sessionID, admitting = "", false + } else { + closing = true + close(completed) + } + } + mu.Unlock() + requests.Done() + }() + handler.ServeHTTP(w, r) + }) + server := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second} + stopped := make(chan struct{}) + var serveErr error + go func() { serveErr = server.Serve(listener); close(stopped) }() + defer func() { + mu.Lock() + closing = true + mu.Unlock() + cancel() + server.Close() + <-stopped + requests.Wait() + log.Info(ctx, "Sandbox SSH server stopped") + }() + if err := json.NewEncoder(output).Encode(Readiness{Port: listener.Addr().(*net.TCPAddr).Port, Token: token}); err != nil { + return fmt.Errorf("report relay readiness: %w", err) + } + log.Infof(ctx, "Sandbox SSH server ready on port %d", listener.Addr().(*net.TCPAddr).Port) + timer := time.NewTimer(opts.ConnectTimeout) + defer timer.Stop() + firstConnection := started + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-firstConnection: + timer.Stop() + firstConnection = nil + case <-timer.C: + mu.Lock() + connected := sessionID != "" && !admitting + if !connected { + closing = true + } + mu.Unlock() + if !connected { + log.Info(ctx, "Sandbox SSH server timed out waiting for its first connection") + return nil + } + case <-completed: + return nil + case <-stopped: + if errors.Is(serveErr, http.ErrServerClosed) { + return nil + } + return serveErr + } + } +} diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go new file mode 100644 index 00000000000..ecb06e1e474 --- /dev/null +++ b/experimental/ssh/sandbox/server_test.go @@ -0,0 +1,370 @@ +package sandbox_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + if os.Getenv("DATABRICKS_TEST_SANDBOX_SSHD") == "1" { + if os.Args[1] == "-t" { + os.Exit(0) + } + _, _ = io.Copy(os.Stdout, os.Stdin) + os.Exit(0) + } + os.Exit(m.Run()) +} + +func startServer(t *testing.T, idle time.Duration) (string, context.CancelFunc, <-chan error, net.Listener) { + t.Helper() + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + reader, writer := io.Pipe() + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: executable, Config: "test-config", ConnectTimeout: idle}, writer) + writer.Close() + }() + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + require.Equal(t, listener.Addr().(*net.TCPAddr).Port, ready.Port) + return "ws://token:" + ready.Token + "@" + listener.Addr().String() + "/ssh", cancel, finished, listener +} + +func TestServerRelaysBinaryDataAndCancels(t *testing.T) { + endpoint, cancel, finished, _ := startServer(t, time.Minute) + conn, err := dialWebSocket(t.Context(), endpoint+"?id=first") + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + payload := bytes.Repeat([]byte{0, 1, 127, 128, 255}, 1000) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, payload)) + var received []byte + for len(received) < len(payload) { + kind, chunk, err := conn.ReadMessage() + require.NoError(t, err) + require.Equal(t, websocket.BinaryMessage, kind) + received = append(received, chunk...) + } + assert.Equal(t, payload, received) + second, response, err := testDial(t.Context(), endpoint+"?id=second") + if second != nil { + second.Close() + } + require.Error(t, err) + require.NotNil(t, response) + assert.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + response.Body.Close() + cancel() + select { + case err := <-finished: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("server did not reap its child on cancellation") + } +} + +func TestServerIdleShutdown(t *testing.T) { + _, _, finished, _ := startServer(t, 50*time.Millisecond) + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("idle server did not stop") + } +} + +func TestServerRejectsInvalidConfigurationBeforeReadiness(t *testing.T) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + var output bytes.Buffer + err = sandbox.Serve(t.Context(), listener, sandbox.ServerOptions{SSHD: "missing-sshd", Config: "config", ConnectTimeout: time.Minute}, &output) + require.ErrorContains(t, err, "validate sshd configuration") + assert.Empty(t, output.String()) + _, err = listener.Accept() + assert.ErrorIs(t, err, net.ErrClosed) +} + +func TestServerStopsAfterItsSessionEnds(t *testing.T) { + endpoint, _, finished, _ := startServer(t, time.Minute) + first, err := dialWebSocket(t.Context(), endpoint+"?id=first") + require.NoError(t, err) + first.Close() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("server did not stop after its SSH session ended") + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken output") } + +func TestServerReadinessWriteFailure(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + err = sandbox.Serve(t.Context(), listener, sandbox.ServerOptions{SSHD: executable, Config: "test-config", ConnectTimeout: time.Minute}, failingWriter{}) + require.ErrorContains(t, err, "report relay readiness") + conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if conn != nil { + conn.Close() + } + require.Error(t, err) +} + +func TestServerListenerFailure(t *testing.T) { + _, _, finished, listener := startServer(t, time.Minute) + require.NoError(t, listener.Close()) + select { + case err := <-finished: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(5 * time.Second): + t.Fatal("server did not return after listener failure") + } +} + +func TestServerRejectsResumeProtocol(t *testing.T) { + resume := fmt.Sprintf("?id=first&delivered=0&resume_version=%d", proxy.ResumeProtocolVersion) + for _, query := range []string{resume, resume + "&reattach=1"} { + t.Run(query, func(t *testing.T) { + endpoint, _, _, _ := startServer(t, time.Minute) + conn, response, err := testDial(t.Context(), endpoint+query) + if conn != nil { + conn.Close() + } + require.Error(t, err) + require.NotNil(t, response) + defer response.Body.Close() + assert.Equal(t, http.StatusBadRequest, response.StatusCode) + + conn, err = dialWebSocket(t.Context(), endpoint+"?id=plain") + require.NoError(t, err) + conn.Close() + }) + } +} + +func TestServerKeepsActiveConnectionPastIdleTimeout(t *testing.T) { + endpoint, cancel, finished, _ := startServer(t, 200*time.Millisecond) + conn, err := dialWebSocket(t.Context(), endpoint+"?id=active") + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, []byte("before"))) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, _, err = conn.ReadMessage() + require.NoError(t, err) + time.Sleep(300 * time.Millisecond) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, []byte("after"))) + _, message, err := conn.ReadMessage() + require.NoError(t, err) + assert.Equal(t, "after", string(message)) + cancel() + require.ErrorIs(t, <-finished, context.Canceled) +} + +func dialWebSocket(ctx context.Context, endpoint string) (*websocket.Conn, error) { + conn, response, err := testDial(ctx, endpoint) + if response != nil { + response.Body.Close() + } + return conn, err +} + +// Blocking readiness lets the idle timer fire before Serve starts receiving it. +type blockedReadiness struct { + entered chan struct{} + release chan struct{} +} + +func (writer blockedReadiness) Write(data []byte) (int, error) { + close(writer.entered) + <-writer.release + return len(data), nil +} + +func TestServerConnectionDeadlineStartsAfterReadiness(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + output := blockedReadiness{entered: make(chan struct{}), release: make(chan struct{})} + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: executable, Config: "config", ConnectTimeout: 50 * time.Millisecond}, output) + }() + <-output.entered + time.Sleep(100 * time.Millisecond) + close(output.release) + select { + case err := <-finished: + require.NoError(t, err) + t.Fatal("connection deadline began before readiness") + case <-time.After(20 * time.Millisecond): + } + require.NoError(t, <-finished) +} + +func testDial(ctx context.Context, endpoint string) (*websocket.Conn, *http.Response, error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, nil, err + } + headers := http.Header{} + if u.User != nil { + token, _ := u.User.Password() + headers.Set(sandbox.RelayTokenHeader, token) + u.User = nil + } + return websocket.DefaultDialer.DialContext(ctx, u.String(), headers) +} + +func TestServerRejectsConnectionsWithoutInvocationToken(t *testing.T) { + endpoint, _, finished, _ := startServer(t, 100*time.Millisecond) + u, err := url.Parse(endpoint) + require.NoError(t, err) + u.User = nil + for _, token := range []string{"", "another-invocation"} { + headers := http.Header{} + headers.Set(sandbox.RelayTokenHeader, token) + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), u.String()+"?id=intruder", headers) + if conn != nil { + conn.Close() + } + require.Error(t, err) + require.NotNil(t, response) + assert.Equal(t, http.StatusUnauthorized, response.StatusCode) + response.Body.Close() + } + require.NoError(t, <-finished) +} + +func TestServerMalformedRequestsDoNotExtendConnectionDeadline(t *testing.T) { + endpoint, _, finished, _ := startServer(t, 100*time.Millisecond) + deadline := time.After(3 * time.Second) + ticker := time.NewTicker(5 * time.Millisecond) + defer ticker.Stop() + for { + select { + case err := <-finished: + require.NoError(t, err) + return + case <-deadline: + t.Fatal("malformed requests kept the server alive") + case <-ticker.C: + conn, response, _ := testDial(t.Context(), endpoint) + if conn != nil { + conn.Close() + } + if response != nil { + response.Body.Close() + } + } + } +} + +func TestServerHandoverSurvivesWebSocketLifetimeLimit(t *testing.T) { + endpoint, cancel, finished, _ := startServer(t, time.Minute) + defer cancel() + target, err := url.Parse(endpoint) + require.NoError(t, err) + token, _ := target.User.Password() + target.User = nil + target.Scheme = "http" + target.Path = "" + forwarding := httptest.NewUnstartedServer(httputil.NewSingleHostReverseProxy(target)) + forwarding.Config.ConnState = func(conn net.Conn, state http.ConnState) { + if state == http.StateHijacked { + time.AfterFunc(500*time.Millisecond, func() { conn.Close() }) + } + } + forwarding.Start() + defer forwarding.Close() + input, send := io.Pipe() + receive, output := io.Pipe() + defer send.Close() + defer receive.Close() + ctx, stop := context.WithCancel(t.Context()) + defer stop() + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + var dials atomic.Int32 + result := make(chan error, 1) + go func() { + result <- proxy.RunClientProxy(ctx, input, output, func() <-chan time.Time { return tick.C }, time.Second, false, + func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + headers := http.Header{} + headers.Set(sandbox.RelayTokenHeader, token) + conn, response, err := websocket.DefaultDialer.DialContext(ctx, "ws"+strings.TrimPrefix(forwarding.URL, "http")+"/ssh?id="+request.ConnID, headers) + if response != nil { + response.Body.Close() + } + if err == nil { + dials.Add(1) + } + return conn, err + }) + }() + for i := range 5 { + payload := []byte(fmt.Sprintf("message-%d", i)) + written := make(chan error, 1) + go func() { _, err := send.Write(payload); written <- err }() + select { + case err := <-written: + require.NoError(t, err) + case err := <-result: + t.Fatalf("client ended before data transfer: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("client did not read SSH input") + } + got := make([]byte, len(payload)) + read := make(chan error, 1) + go func() { _, err := io.ReadFull(receive, got); read <- err }() + select { + case err := <-read: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("SSH stream stopped during handover") + } + require.Equal(t, payload, got) + time.Sleep(200 * time.Millisecond) + } + require.GreaterOrEqual(t, dials.Load(), int32(3)) + stop() + require.NoError(t, <-result) + cancel() + require.ErrorIs(t, <-finished, context.Canceled) +} From 44a89dddfdcd54278029d5bf833c1ba6b7743cac Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Thu, 19 Nov 2026 03:50:47 +0000 Subject: [PATCH 2/7] Simplify sandbox session ownership and strengthen shutdown coverage --- acceptance/cmd/sandbox/server/server_test.py | 19 ++++++-- acceptance/cmd/sandbox/server/test.toml | 1 - .../ssh/internal/proxy/client_server_test.go | 11 +++-- .../ssh/internal/proxy/connections.go | 12 +---- .../ssh/internal/proxy/connections_test.go | 13 ------ experimental/ssh/internal/proxy/server.go | 21 ++++----- .../ssh/internal/proxy/server_options_test.go | 21 +++------ experimental/ssh/sandbox/server.go | 44 +++++-------------- experimental/ssh/sandbox/server_test.go | 43 ++++++++++++++++-- 9 files changed, 96 insertions(+), 89 deletions(-) diff --git a/acceptance/cmd/sandbox/server/server_test.py b/acceptance/cmd/sandbox/server/server_test.py index 1d4589d4428..bfcd410a06f 100644 --- a/acceptance/cmd/sandbox/server/server_test.py +++ b/acceptance/cmd/sandbox/server/server_test.py @@ -22,7 +22,9 @@ child_pid = directory / "child.pid" sshd = directory / "sshd" # A long-running child exercises cancellation without requiring system sshd. - sshd.write_text('#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\necho $$ > "' + str(child_pid) + '"\nexec cat\n') + sshd.write_text( + '#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\necho $$ > "' + str(child_pid) + '"\nexec sleep 300\n' + ) sshd.chmod(0o700) help_result = subprocess.run( [cli, "sandbox", "--help"], env=environment, capture_output=True, text=True, check=True @@ -35,6 +37,7 @@ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + start_new_session=True, ) try: ready = json.loads(process.stdout.readline()) @@ -56,6 +59,8 @@ assert time.monotonic() < deadline, "sshd child did not start" time.sleep(0.01) pid = int(child_pid.read_text()) + os.kill(pid, 0) + assert process.poll() is None, "server exited before SIGTERM" process.send_signal(signal.SIGTERM) assert process.wait(timeout=10) == 1 # context cancellation, rather than signal termination try: @@ -67,5 +72,13 @@ print("SIGTERM stops the server and its SSH child") finally: if process.poll() is None: - process.kill() - process.wait() + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass diff --git a/acceptance/cmd/sandbox/server/test.toml b/acceptance/cmd/sandbox/server/test.toml index e0829cb064d..af216b4f2de 100644 --- a/acceptance/cmd/sandbox/server/test.toml +++ b/acceptance/cmd/sandbox/server/test.toml @@ -1,4 +1,3 @@ # The server and its signal handling run inside Linux sandboxes. GOOS.windows = false GOOS.darwin = false -Ignore = [".databricks", ".ssh", "empty-home", "child.pid"] diff --git a/experimental/ssh/internal/proxy/client_server_test.go b/experimental/ssh/internal/proxy/client_server_test.go index 6ed84ddbce2..9680c68ac88 100644 --- a/experimental/ssh/internal/proxy/client_server_test.go +++ b/experimental/ssh/internal/proxy/client_server_test.go @@ -565,12 +565,12 @@ func TestClientTimesOutWhenServerSendsNothing(t *testing.T) { func TestSingleSessionCannotStartAgainAfterDisconnect(t *testing.T) { manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) - defer manager.Close() + completed := make(chan struct{}) var started atomic.Int32 handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { started.Add(1) return exec.CommandContext(ctx, "cat") - }, WithSingleSession()) + }, WithSingleSession(), WithSessionCompletion(func() { close(completed) })) server := httptest.NewServer(handler) defer server.Close() connection, response, err := websocket.DefaultDialer.Dial("ws"+server.URL[4:]+"?id=session", nil) @@ -584,7 +584,12 @@ func TestSingleSessionCannotStartAgainAfterDisconnect(t *testing.T) { require.NoError(t, err) require.Equal(t, "first-session", string(message)) require.NoError(t, connection.Close()) - require.Eventually(t, func() bool { return manager.Count() == 0 }, 5*time.Second, time.Millisecond) + select { + case <-completed: + case <-time.After(5 * time.Second): + t.Fatal("logical session completion was not reported") + } + require.Zero(t, manager.Count()) for _, id := range []string{"session", "another-session"} { response := httptest.NewRecorder() handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id="+id, nil)) diff --git a/experimental/ssh/internal/proxy/connections.go b/experimental/ssh/internal/proxy/connections.go index b025528a071..1a5ef07acd4 100644 --- a/experimental/ssh/internal/proxy/connections.go +++ b/experimental/ssh/internal/proxy/connections.go @@ -31,7 +31,7 @@ func NewConnectionsManager(maxClients int, shutdownDelay time.Duration, options maxClients: maxClients, shutdownDelay: shutdownDelay, connections: make(map[string]*proxyConnection), - TimedOut: make(chan bool, 1), + TimedOut: make(chan bool), } for _, option := range options { option(cm) @@ -101,10 +101,7 @@ func (cm *ConnectionsManager) startShutdownTimer(delay time.Duration) { cm.shutdownTimer.Stop() } cm.shutdownTimer = time.AfterFunc(delay, func() { - select { - case cm.TimedOut <- true: - default: - } + cm.TimedOut <- true }) } @@ -115,8 +112,3 @@ func (cm *ConnectionsManager) cancelShutdownTimer() { cm.shutdownTimer.Stop() } } - -// Close stops the idle timer. -func (cm *ConnectionsManager) Close() { - cm.cancelShutdownTimer() -} diff --git a/experimental/ssh/internal/proxy/connections_test.go b/experimental/ssh/internal/proxy/connections_test.go index a55ed3ebe1c..e05c92d1fcb 100644 --- a/experimental/ssh/internal/proxy/connections_test.go +++ b/experimental/ssh/internal/proxy/connections_test.go @@ -239,16 +239,3 @@ func TestConnectionsManager_ShutdownTimer_NoRestartWhenConnectionsRemain(t *test assert.False(t, timedOut, "Unexpected timeout signal while connections still exist") }) } - -func TestConnectionsManagerCloseStopsIdleTimer(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - cm := NewConnectionsManager(1, time.Second) - cm.Close() - time.Sleep(2 * time.Second) - select { - case <-cm.TimedOut: - t.Fatal("closed manager timed out") - default: - } - }) -} diff --git a/experimental/ssh/internal/proxy/server.go b/experimental/ssh/internal/proxy/server.go index dde33c00d5d..88c3d270794 100644 --- a/experimental/ssh/internal/proxy/server.go +++ b/experimental/ssh/internal/proxy/server.go @@ -25,7 +25,7 @@ type proxyServer struct { createServerCommand createServerCommandFunc resumeGrace time.Duration allowResume bool - allowHandover bool + onSessionEnd func() singleSession bool sessionUsed bool sessionMu sync.Mutex @@ -39,9 +39,10 @@ func WithoutSessionResume() ServerOption { return func(server *proxyServer) { server.allowResume = false } } -// WithoutHandover rejects requests to switch an active session to another WebSocket. -func WithoutHandover() ServerOption { - return func(server *proxyServer) { server.allowHandover = false } +// WithSessionCompletion runs notify after an accepted logical session ends and is removed. +// Handovers and failed upgrades do not trigger it. +func WithSessionCompletion(notify func()) ServerOption { + return func(server *proxyServer) { server.onSessionEnd = notify } } // WithSingleSession permits handovers but never starts a second SSH session. @@ -56,7 +57,6 @@ func NewProxyServer(ctx context.Context, connections *ConnectionsManager, create createServerCommand: createServerCommand, resumeGrace: proxyResumeGrace, allowResume: true, - allowHandover: true, } for _, option := range options { option(server) @@ -85,10 +85,6 @@ func (server *proxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { case exists && conn != nil && req.Reattach: server.handleReattach(ctx, w, r, conn, req) case exists && conn != nil: - if !server.allowHandover { - http.Error(w, "Session handover is not supported", http.StatusBadRequest) - return - } server.handleExistingConnection(ctx, w, r, conn) case req.Reattach: // The session is gone: its grace period expired, or the server restarted. Say so instead @@ -189,7 +185,12 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo http.Error(w, "Maximum clients reached", http.StatusServiceUnavailable) return } - defer server.connections.Remove(id) + defer func() { + server.connections.Remove(id) + if accepted && server.onSessionEnd != nil { + server.onSessionEnd() + } + }() log.Infof(ctx, "Starting proxy server for new connection, count: %d", server.connections.Count()) err := runServerProxy(ctx, conn, func(ctx context.Context) *exec.Cmd { diff --git a/experimental/ssh/internal/proxy/server_options_test.go b/experimental/ssh/internal/proxy/server_options_test.go index 99a98ad535c..1131eaee093 100644 --- a/experimental/ssh/internal/proxy/server_options_test.go +++ b/experimental/ssh/internal/proxy/server_options_test.go @@ -12,27 +12,20 @@ import ( "github.com/stretchr/testify/require" ) -func TestServerCanDisableResumeAndHandover(t *testing.T) { +func TestServerCanDisableResume(t *testing.T) { manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) - defer manager.Close() handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { t.Error("rejected requests must not start an SSH process") return exec.CommandContext(ctx, "unused") - }, WithoutSessionResume(), WithoutHandover()) + }, WithoutSessionResume()) response := httptest.NewRecorder() handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/ssh?id=test&delivered=0&resume_version=%d", ResumeProtocolVersion), nil)) require.Equal(t, http.StatusBadRequest, response.Code) require.Contains(t, response.Body.String(), "resume is not supported") - require.True(t, manager.TryAdd("test", newProxyConnection(nil))) - response = httptest.NewRecorder() - handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id=test", nil)) - require.Equal(t, http.StatusBadRequest, response.Code) - require.Contains(t, response.Body.String(), "handover is not supported") } func TestManagerCanDisableIdleShutdown(t *testing.T) { manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) - defer manager.Close() require.True(t, manager.TryAdd("session", newProxyConnection(nil))) manager.Remove("session") select { @@ -44,12 +37,11 @@ func TestManagerCanDisableIdleShutdown(t *testing.T) { func TestSingleSessionRejectsReuseAfterManagerRemoval(t *testing.T) { manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) - defer manager.Close() handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { t.Fatal("a completed logical session must never start another process") return nil }, WithSingleSession()) - // Reproduce the gap after Remove and before the outer sandbox handler returns. + // Simulate a session that already ended and was removed from the manager. handler.sessionUsed = true for _, id := range []string{"same-session", "different-session"} { response := httptest.NewRecorder() @@ -59,23 +51,24 @@ func TestSingleSessionRejectsReuseAfterManagerRemoval(t *testing.T) { } func TestSingleSessionFailedUpgradeDoesNotConsumeSession(t *testing.T) { + completed := false manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) - defer manager.Close() handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { t.Fatal("a non-WebSocket request must never start a process") return nil - }, WithSingleSession()) + }, WithSingleSession(), WithSessionCompletion(func() { completed = true })) for range 2 { response := httptest.NewRecorder() handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id=test", nil)) require.Equal(t, http.StatusBadRequest, response.Code) require.False(t, handler.sessionUsed) + require.False(t, completed) + require.Zero(t, manager.Count()) } } func TestZeroIdleTimeoutStillSignalsImmediately(t *testing.T) { manager := NewConnectionsManager(1, 0) - defer manager.Close() select { case <-manager.TimedOut: case <-time.After(5 * time.Second): diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go index 0a50eb0395f..47b59a28209 100644 --- a/experimental/ssh/sandbox/server.go +++ b/experimental/ssh/sandbox/server.go @@ -52,20 +52,17 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu ctx, cancel := context.WithCancel(ctx) defer cancel() connections := proxy.NewConnectionsManager(1, 0, proxy.WithoutIdleShutdown()) - defer connections.Close() token := rand.Text() started, completed := make(chan struct{}), make(chan struct{}) var requests sync.WaitGroup var mu sync.Mutex - var sessionID string - admitting, closing := false, false + closing := false handler := proxy.NewProxyServer(ctx, connections, func(ctx context.Context) *exec.Cmd { mu.Lock() - admitting = false close(started) mu.Unlock() return exec.CommandContext(ctx, opts.SSHD, "-i", "-f", opts.Config) - }, proxy.WithoutSessionResume(), proxy.WithSingleSession()) + }, proxy.WithoutSessionResume(), proxy.WithSingleSession(), proxy.WithSessionCompletion(func() { close(completed) })) mux := http.NewServeMux() mux.HandleFunc("GET /ssh", func(w http.ResponseWriter, r *http.Request) { if subtle.ConstantTimeCompare([]byte(r.Header.Get(RelayTokenHeader)), []byte(token)) != 1 { @@ -73,37 +70,16 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu http.Error(w, "Unauthorized", http.StatusUnauthorized) return } - id := r.URL.Query().Get("id") - if id == "" { - http.Error(w, "Missing 'id' query parameter", http.StatusBadRequest) - return - } mu.Lock() - if closing || admitting || (sessionID != "" && sessionID != id) { + if closing { mu.Unlock() - log.Info(ctx, "Rejecting another sandbox SSH session") - http.Error(w, "SSH session already active or server is stopping", http.StatusServiceUnavailable) + log.Info(ctx, "Rejecting sandbox SSH connection while stopping") + http.Error(w, "SSH server is stopping", http.StatusServiceUnavailable) return } - first := sessionID == "" - if first { - sessionID, admitting = id, true - } requests.Add(1) mu.Unlock() - defer func() { - mu.Lock() - if first { - if admitting { - sessionID, admitting = "", false - } else { - closing = true - close(completed) - } - } - mu.Unlock() - requests.Done() - }() + defer requests.Done() handler.ServeHTTP(w, r) }) server := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second} @@ -136,9 +112,13 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu firstConnection = nil case <-timer.C: mu.Lock() - connected := sessionID != "" && !admitting - if !connected { + connected := false + select { + case <-started: + connected = true + default: closing = true + cancel() } mu.Unlock() if !connected { diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index ecb06e1e474..5200a85dc96 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -201,7 +201,7 @@ func dialWebSocket(ctx context.Context, endpoint string) (*websocket.Conn, error return conn, err } -// Blocking readiness lets the idle timer fire before Serve starts receiving it. +// Blocking readiness past ConnectTimeout catches a deadline that starts before readiness is written. type blockedReadiness struct { entered chan struct{} release chan struct{} @@ -365,6 +365,43 @@ func TestServerHandoverSurvivesWebSocketLifetimeLimit(t *testing.T) { require.GreaterOrEqual(t, dials.Load(), int32(3)) stop() require.NoError(t, <-result) - cancel() - require.ErrorIs(t, <-finished, context.Canceled) + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("server did not stop after its handed-over session ended") + } +} + +func TestServerFailedUpgradeCanRetry(t *testing.T) { + endpoint, _, finished, _ := startServer(t, time.Minute) + u, err := url.Parse(endpoint) + require.NoError(t, err) + token, _ := u.User.Password() + u.User = nil + u.Scheme = "http" + u.RawQuery = "id=failed" + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, u.String(), nil) + require.NoError(t, err) + request.Header.Set(sandbox.RelayTokenHeader, token) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + response.Body.Close() + require.Equal(t, http.StatusBadRequest, response.StatusCode) + + conn, err := dialWebSocket(t.Context(), endpoint+"?id=retry") + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, []byte("retry"))) + _, message, err := conn.ReadMessage() + require.NoError(t, err) + require.Equal(t, "retry", string(message)) + require.NoError(t, conn.Close()) + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("server did not stop after the retried session ended") + } } From 6ac378899c2c93473a38eb3ba01320d22fcef124 Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Thu, 19 Nov 2026 04:38:58 +0000 Subject: [PATCH 3/7] Reuse SSH session resume for sandbox WebSocket interruptions --- experimental/ssh/internal/proxy/server.go | 11 --- .../ssh/internal/proxy/server_options_test.go | 13 --- experimental/ssh/sandbox/server.go | 9 +- experimental/ssh/sandbox/server_test.go | 99 ++++++++++++++++++- 4 files changed, 101 insertions(+), 31 deletions(-) diff --git a/experimental/ssh/internal/proxy/server.go b/experimental/ssh/internal/proxy/server.go index 88c3d270794..44e8a24bf25 100644 --- a/experimental/ssh/internal/proxy/server.go +++ b/experimental/ssh/internal/proxy/server.go @@ -24,7 +24,6 @@ type proxyServer struct { connections *ConnectionsManager createServerCommand createServerCommandFunc resumeGrace time.Duration - allowResume bool onSessionEnd func() singleSession bool sessionUsed bool @@ -34,11 +33,6 @@ type proxyServer struct { // ServerOption configures the SSH transport capabilities. type ServerOption func(*proxyServer) -// WithoutSessionResume rejects requests to resume an interrupted session. -func WithoutSessionResume() ServerOption { - return func(server *proxyServer) { server.allowResume = false } -} - // WithSessionCompletion runs notify after an accepted logical session ends and is removed. // Handovers and failed upgrades do not trigger it. func WithSessionCompletion(notify func()) ServerOption { @@ -56,7 +50,6 @@ func NewProxyServer(ctx context.Context, connections *ConnectionsManager, create connections: connections, createServerCommand: createServerCommand, resumeGrace: proxyResumeGrace, - allowResume: true, } for _, option := range options { option(server) @@ -75,10 +68,6 @@ func (server *proxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - if !server.allowResume && (req.ResumeCapable || req.Reattach) { - http.Error(w, "Session resume is not supported", http.StatusBadRequest) - return - } ctx := log.NewContext(server.ctx, log.GetLogger(server.ctx).With("session", id)) conn, exists := server.connections.Get(id) switch { diff --git a/experimental/ssh/internal/proxy/server_options_test.go b/experimental/ssh/internal/proxy/server_options_test.go index 1131eaee093..00083230e03 100644 --- a/experimental/ssh/internal/proxy/server_options_test.go +++ b/experimental/ssh/internal/proxy/server_options_test.go @@ -2,7 +2,6 @@ package proxy import ( "context" - "fmt" "net/http" "net/http/httptest" "os/exec" @@ -12,18 +11,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestServerCanDisableResume(t *testing.T) { - manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) - handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { - t.Error("rejected requests must not start an SSH process") - return exec.CommandContext(ctx, "unused") - }, WithoutSessionResume()) - response := httptest.NewRecorder() - handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/ssh?id=test&delivered=0&resume_version=%d", ResumeProtocolVersion), nil)) - require.Equal(t, http.StatusBadRequest, response.Code) - require.Contains(t, response.Body.String(), "resume is not supported") -} - func TestManagerCanDisableIdleShutdown(t *testing.T) { manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) require.True(t, manager.TryAdd("session", newProxyConnection(nil))) diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go index 47b59a28209..2c770247eef 100644 --- a/experimental/ssh/sandbox/server.go +++ b/experimental/ssh/sandbox/server.go @@ -31,8 +31,9 @@ type ServerOptions struct { // Readiness is written as one JSON line after the sshd configuration is validated. // Token is only returned over the caller's authenticated command-output stream. type Readiness struct { - Port int `json:"port"` - Token string `json:"token"` + Port int `json:"port"` + Token string `json:"token"` + ResumeVersion int `json:"resume_version"` } // Serve relays /ssh WebSockets to one sshd session, including its handovers. @@ -62,7 +63,7 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu close(started) mu.Unlock() return exec.CommandContext(ctx, opts.SSHD, "-i", "-f", opts.Config) - }, proxy.WithoutSessionResume(), proxy.WithSingleSession(), proxy.WithSessionCompletion(func() { close(completed) })) + }, proxy.WithSingleSession(), proxy.WithSessionCompletion(func() { close(completed) })) mux := http.NewServeMux() mux.HandleFunc("GET /ssh", func(w http.ResponseWriter, r *http.Request) { if subtle.ConstantTimeCompare([]byte(r.Header.Get(RelayTokenHeader)), []byte(token)) != 1 { @@ -96,7 +97,7 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu requests.Wait() log.Info(ctx, "Sandbox SSH server stopped") }() - if err := json.NewEncoder(output).Encode(Readiness{Port: listener.Addr().(*net.TCPAddr).Port, Token: token}); err != nil { + if err := json.NewEncoder(output).Encode(Readiness{Port: listener.Addr().(*net.TCPAddr).Port, Token: token, ResumeVersion: proxy.ResumeProtocolVersion}); err != nil { return fmt.Errorf("report relay readiness: %w", err) } log.Infof(ctx, "Sandbox SSH server ready on port %d", listener.Addr().(*net.TCPAddr).Port) diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index 5200a85dc96..07b106892ee 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -1,6 +1,7 @@ package sandbox_test import ( + "bufio" "bytes" "context" "encoding/json" @@ -30,6 +31,9 @@ func TestMain(m *testing.M) { if os.Args[1] == "-t" { os.Exit(0) } + if os.Getenv("DATABRICKS_TEST_SANDBOX_SSHD_PID") == "1" { + fmt.Fprintf(os.Stdout, "pid:%d\n", os.Getpid()) + } _, _ = io.Copy(os.Stdout, os.Stdin) os.Exit(0) } @@ -54,6 +58,7 @@ func startServer(t *testing.T, idle time.Duration) (string, context.CancelFunc, var ready sandbox.Readiness require.NoError(t, json.NewDecoder(reader).Decode(&ready)) require.Equal(t, listener.Addr().(*net.TCPAddr).Port, ready.Port) + require.Equal(t, proxy.ResumeProtocolVersion, ready.ResumeVersion) return "ws://token:" + ready.Token + "@" + listener.Addr().String() + "/ssh", cancel, finished, listener } @@ -154,9 +159,8 @@ func TestServerListenerFailure(t *testing.T) { } } -func TestServerRejectsResumeProtocol(t *testing.T) { - resume := fmt.Sprintf("?id=first&delivered=0&resume_version=%d", proxy.ResumeProtocolVersion) - for _, query := range []string{resume, resume + "&reattach=1"} { +func TestServerRejectsUnsupportedResumeProtocol(t *testing.T) { + for _, query := range []string{"?id=first&delivered=0&resume_version=1", "?id=first&reattach=1"} { t.Run(query, func(t *testing.T) { endpoint, _, _, _ := startServer(t, time.Minute) conn, response, err := testDial(t.Context(), endpoint+query) @@ -175,6 +179,95 @@ func TestServerRejectsResumeProtocol(t *testing.T) { } } +func TestServerRejectsResumeOfMissingSession(t *testing.T) { + endpoint, _, _, _ := startServer(t, time.Minute) + conn, response, err := testDial(t.Context(), resumeEndpoint(endpoint, proxy.DialRequest{ConnID: "missing", ResumeCapable: true, Reattach: true})) + if conn != nil { + conn.Close() + } + require.Error(t, err) + require.NotNil(t, response) + defer response.Body.Close() + assert.Equal(t, http.StatusGone, response.StatusCode) +} + +func resumeEndpoint(endpoint string, request proxy.DialRequest) string { + query := url.Values{"id": {request.ConnID}} + if request.ResumeCapable { + query.Set("delivered", fmt.Sprint(request.Delivered)) + query.Set(proxy.ResumeVersionParameter, fmt.Sprint(proxy.ResumeProtocolVersion)) + } + if request.Reattach { + query.Set("reattach", "1") + } + return endpoint + "?" + query.Encode() +} + +func TestServerResumesSameProcessAfterWebSocketDrops(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD_PID", "1") + endpoint, cancel, finished, _ := startServer(t, time.Minute) + defer cancel() + input, send := io.Pipe() + receive, output := io.Pipe() + defer send.Close() + defer receive.Close() + ctx, stop := context.WithCancel(t.Context()) + defer stop() + connections := make(chan *websocket.Conn, 8) + var reattachments atomic.Int32 + result := make(chan error, 1) + go func() { + result <- proxy.RunClientProxy(ctx, input, output, func() <-chan time.Time { return nil }, time.Second, true, func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + conn, err := dialWebSocket(ctx, resumeEndpoint(endpoint, request)) + if err == nil { + connections <- conn + if request.Reattach { + reattachments.Add(1) + } + } + return conn, err + }) + output.Close() + }() + reader := bufio.NewReader(receive) + pid, err := reader.ReadString('\n') + require.NoError(t, err) + require.Regexp(t, `^pid:[0-9]+\n$`, pid) + for round := range 3 { + var connection *websocket.Conn + select { + case connection = <-connections: + case <-time.After(5 * time.Second): + t.Fatal("client did not connect") + } + payload := bytes.Repeat([]byte{byte(round), 0, 127, 128, 255}, 10000) + written := make(chan error, 1) + go func() { _, err := send.Write(payload); written <- err }() + require.NoError(t, connection.UnderlyingConn().Close()) + got := make([]byte, len(payload)) + read := make(chan error, 1) + go func() { _, err := io.ReadFull(reader, got); read <- err }() + select { + case err := <-read: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("SSH output did not resume after the WebSocket dropped") + } + require.NoError(t, <-written) + // A restarted child emits another PID line and corrupts this comparison. + require.Equal(t, payload, got) + } + require.Equal(t, int32(3), reattachments.Load()) + stop() + require.NoError(t, <-result) + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("server did not stop after its resumed session ended") + } +} + func TestServerKeepsActiveConnectionPastIdleTimeout(t *testing.T) { endpoint, cancel, finished, _ := startServer(t, 200*time.Millisecond) conn, err := dialWebSocket(t.Context(), endpoint+"?id=active") From 154b19cb4de33fa21184b37336ed2191e86aefd7 Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Thu, 19 Nov 2026 04:48:00 +0000 Subject: [PATCH 4/7] Use integer conversion helpers in resume regression tests --- experimental/ssh/sandbox/server_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index 07b106892ee..44f62d85868 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -14,6 +14,7 @@ import ( "net/http/httputil" "net/url" "os" + "strconv" "strings" "sync/atomic" "testing" @@ -194,8 +195,8 @@ func TestServerRejectsResumeOfMissingSession(t *testing.T) { func resumeEndpoint(endpoint string, request proxy.DialRequest) string { query := url.Values{"id": {request.ConnID}} if request.ResumeCapable { - query.Set("delivered", fmt.Sprint(request.Delivered)) - query.Set(proxy.ResumeVersionParameter, fmt.Sprint(proxy.ResumeProtocolVersion)) + query.Set("delivered", strconv.FormatInt(request.Delivered, 10)) + query.Set(proxy.ResumeVersionParameter, strconv.Itoa(proxy.ResumeProtocolVersion)) } if request.Reattach { query.Set("reattach", "1") From 219d06c0e841c61e24dc682cb3ffbe03dc9a3685 Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Thu, 19 Nov 2026 05:09:48 +0000 Subject: [PATCH 5/7] Report sshd startup failures without consuming the SSH session --- .../ssh/internal/proxy/client_server_test.go | 2 +- experimental/ssh/internal/proxy/server.go | 33 +++++++----- .../ssh/internal/proxy/server_options_test.go | 53 ++++++++++++++++++- experimental/ssh/sandbox/server.go | 9 ++-- experimental/ssh/sandbox/server_test.go | 34 ++++++++++++ 5 files changed, 113 insertions(+), 18 deletions(-) diff --git a/experimental/ssh/internal/proxy/client_server_test.go b/experimental/ssh/internal/proxy/client_server_test.go index 9680c68ac88..bcb0771de62 100644 --- a/experimental/ssh/internal/proxy/client_server_test.go +++ b/experimental/ssh/internal/proxy/client_server_test.go @@ -570,7 +570,7 @@ func TestSingleSessionCannotStartAgainAfterDisconnect(t *testing.T) { handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { started.Add(1) return exec.CommandContext(ctx, "cat") - }, WithSingleSession(), WithSessionCompletion(func() { close(completed) })) + }, WithSingleSession(), WithSessionCompletion(func(error) { close(completed) })) server := httptest.NewServer(handler) defer server.Close() connection, response, err := websocket.DefaultDialer.Dial("ws"+server.URL[4:]+"?id=session", nil) diff --git a/experimental/ssh/internal/proxy/server.go b/experimental/ssh/internal/proxy/server.go index 44e8a24bf25..b566743ae74 100644 --- a/experimental/ssh/internal/proxy/server.go +++ b/experimental/ssh/internal/proxy/server.go @@ -24,7 +24,7 @@ type proxyServer struct { connections *ConnectionsManager createServerCommand createServerCommandFunc resumeGrace time.Duration - onSessionEnd func() + onSessionEnd func(error) singleSession bool sessionUsed bool sessionMu sync.Mutex @@ -33,9 +33,9 @@ type proxyServer struct { // ServerOption configures the SSH transport capabilities. type ServerOption func(*proxyServer) -// WithSessionCompletion runs notify after an accepted logical session ends and is removed. -// Handovers and failed upgrades do not trigger it. -func WithSessionCompletion(notify func()) ServerOption { +// WithSessionCompletion reports process startup failures or a completed logical session. +// Established sessions report nil; handovers and failed upgrades do not notify. +func WithSessionCompletion(notify func(error)) ServerOption { return func(server *proxyServer) { server.onSessionEnd = notify } } @@ -140,8 +140,8 @@ func (server *proxyServer) handleExistingConnection(ctx context.Context, w http. } func (server *proxyServer) handleNewConnection(ctx context.Context, w http.ResponseWriter, r *http.Request, id string, req DialRequest) { - // Failed upgrades can retry; a started SSH session consumes this server permanently. - accepted := false + // Failed upgrades and process starts can retry; a started session consumes this server. + processStarted := false if server.singleSession { server.sessionMu.Lock() if server.sessionUsed { @@ -152,7 +152,7 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo server.sessionUsed = true server.sessionMu.Unlock() defer func() { - if !accepted { + if !processStarted { server.sessionMu.Lock() server.sessionUsed = false server.sessionMu.Unlock() @@ -174,18 +174,25 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo http.Error(w, "Maximum clients reached", http.StatusServiceUnavailable) return } + var command *exec.Cmd + var err error defer func() { server.connections.Remove(id) - if accepted && server.onSessionEnd != nil { - server.onSessionEnd() + if command != nil && server.onSessionEnd != nil { + var startErr error + if !processStarted { + startErr = err + } + server.onSessionEnd(startErr) } }() log.Infof(ctx, "Starting proxy server for new connection, count: %d", server.connections.Count()) - err := runServerProxy(ctx, conn, func(ctx context.Context) *exec.Cmd { - accepted = true - return server.createServerCommand(ctx) + err = runServerProxy(ctx, conn, func(ctx context.Context) *exec.Cmd { + command = server.createServerCommand(ctx) + return command }, w, r) + processStarted = command != nil && command.Process != nil if err != nil { log.Errorf(ctx, "Proxy server error: %v", err) } else { @@ -212,11 +219,13 @@ func runServerProxy(ctx context.Context, proxy *proxyConnection, createServerCom if err != nil { return fmt.Errorf("failed to get stdin pipe: %w", err) } + defer sshdStdin.Close() sshdStdout, err := serverCmd.StdoutPipe() if err != nil { return fmt.Errorf("failed to get stdout pipe: %w", err) } + defer sshdStdout.Close() err = serverCmd.Start() if err != nil { diff --git a/experimental/ssh/internal/proxy/server_options_test.go b/experimental/ssh/internal/proxy/server_options_test.go index 00083230e03..cb0357ea078 100644 --- a/experimental/ssh/internal/proxy/server_options_test.go +++ b/experimental/ssh/internal/proxy/server_options_test.go @@ -2,15 +2,66 @@ package proxy import ( "context" + "io" "net/http" "net/http/httptest" "os/exec" + "strings" "testing" "time" + "github.com/gorilla/websocket" "github.com/stretchr/testify/require" ) +func TestSingleSessionReportsStartupFailureAndAllowsRetry(t *testing.T) { + for _, tc := range []struct{ stage, errorText string }{ + {"stdin", "failed to get stdin pipe"}, + {"stdout", "failed to get stdout pipe"}, + {"start", "failed to start SSHD process"}, + } { + t.Run(tc.stage, func(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + completed := make(chan error, 1) + handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { + cmd := exec.CommandContext(ctx, "missing-sshd-executable") + switch tc.stage { + case "stdin": + cmd.Stdin = strings.NewReader("") + case "stdout": + cmd.Stdout = io.Discard + } + return cmd + }, WithSingleSession(), WithSessionCompletion(func(err error) { completed <- err })) + handled := make(chan struct{}, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler.ServeHTTP(w, r) + handled <- struct{}{} + })) + defer server.Close() + for range 2 { + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), "ws"+server.URL[4:]+"/ssh?id=retry", nil) + if response != nil { + response.Body.Close() + } + require.NoError(t, err, "startup failure must not consume the single session") + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, _, err = conn.ReadMessage() + conn.Close() + require.Error(t, err) + select { + case <-handled: + case <-time.After(5 * time.Second): + t.Fatal("failed startup did not release its handler") + } + require.ErrorContains(t, <-completed, tc.errorText) + require.False(t, handler.sessionUsed) + require.Zero(t, manager.Count()) + } + }) + } +} + func TestManagerCanDisableIdleShutdown(t *testing.T) { manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) require.True(t, manager.TryAdd("session", newProxyConnection(nil))) @@ -43,7 +94,7 @@ func TestSingleSessionFailedUpgradeDoesNotConsumeSession(t *testing.T) { handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { t.Fatal("a non-WebSocket request must never start a process") return nil - }, WithSingleSession(), WithSessionCompletion(func() { completed = true })) + }, WithSingleSession(), WithSessionCompletion(func(error) { completed = true })) for range 2 { response := httptest.NewRecorder() handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id=test", nil)) diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go index 2c770247eef..9f6cab4f9fd 100644 --- a/experimental/ssh/sandbox/server.go +++ b/experimental/ssh/sandbox/server.go @@ -54,7 +54,8 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu defer cancel() connections := proxy.NewConnectionsManager(1, 0, proxy.WithoutIdleShutdown()) token := rand.Text() - started, completed := make(chan struct{}), make(chan struct{}) + started := make(chan struct{}) + completed := make(chan error, 1) var requests sync.WaitGroup var mu sync.Mutex closing := false @@ -63,7 +64,7 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu close(started) mu.Unlock() return exec.CommandContext(ctx, opts.SSHD, "-i", "-f", opts.Config) - }, proxy.WithSingleSession(), proxy.WithSessionCompletion(func() { close(completed) })) + }, proxy.WithSingleSession(), proxy.WithSessionCompletion(func(err error) { completed <- err })) mux := http.NewServeMux() mux.HandleFunc("GET /ssh", func(w http.ResponseWriter, r *http.Request) { if subtle.ConstantTimeCompare([]byte(r.Header.Get(RelayTokenHeader)), []byte(token)) != 1 { @@ -126,8 +127,8 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu log.Info(ctx, "Sandbox SSH server timed out waiting for its first connection") return nil } - case <-completed: - return nil + case err := <-completed: + return err case <-stopped: if errors.Is(serveErr, http.ErrServerClosed) { return nil diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index 44f62d85868..5beb65ca54c 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -14,6 +14,8 @@ import ( "net/http/httputil" "net/url" "os" + "path/filepath" + "runtime" "strconv" "strings" "sync/atomic" @@ -117,6 +119,38 @@ func TestServerRejectsInvalidConfigurationBeforeReadiness(t *testing.T) { assert.ErrorIs(t, err, net.ErrClosed) } +func TestServerReportsSSHDStartFailureAfterReadiness(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sandbox servers run on Linux") + } + sshd := filepath.Join(t.TempDir(), "sshd") + require.NoError(t, os.WriteFile(sshd, []byte("#!/bin/sh\nexit 0\n"), 0o700)) + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader, writer := io.Pipe() + defer reader.Close() + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: sshd, Config: "config", ConnectTimeout: time.Minute}, writer) + writer.Close() + }() + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + // Force a real exec failure after the configuration check succeeded. + require.NoError(t, os.Remove(sshd)) + conn, err := dialWebSocket(t.Context(), "ws://token:"+ready.Token+"@"+listener.Addr().String()+"/ssh?id=failed-start") + require.NoError(t, err) + defer conn.Close() + select { + case err := <-finished: + require.ErrorContains(t, err, "failed to start SSHD process") + case <-time.After(5 * time.Second): + t.Fatal("sshd startup failure did not stop the server") + } +} + func TestServerStopsAfterItsSessionEnds(t *testing.T) { endpoint, _, finished, _ := startServer(t, time.Minute) first, err := dialWebSocket(t.Context(), endpoint+"?id=first") From 605f0a01f206338f5568b963c894a32b48ebaed2 Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Thu, 19 Nov 2026 16:08:38 +0000 Subject: [PATCH 6/7] Keep single-session admission consumed after sshd startup failure --- experimental/ssh/internal/proxy/server.go | 10 +-- .../ssh/internal/proxy/server_options_test.go | 78 ++++++++++++++----- experimental/ssh/sandbox/server_test.go | 17 +++- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/experimental/ssh/internal/proxy/server.go b/experimental/ssh/internal/proxy/server.go index b566743ae74..59a21c9d302 100644 --- a/experimental/ssh/internal/proxy/server.go +++ b/experimental/ssh/internal/proxy/server.go @@ -140,8 +140,8 @@ func (server *proxyServer) handleExistingConnection(ctx context.Context, w http. } func (server *proxyServer) handleNewConnection(ctx context.Context, w http.ResponseWriter, r *http.Request, id string, req DialRequest) { - // Failed upgrades and process starts can retry; a started session consumes this server. - processStarted := false + // Failed upgrades can retry; invoking the process factory consumes this server. + var command *exec.Cmd if server.singleSession { server.sessionMu.Lock() if server.sessionUsed { @@ -152,7 +152,7 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo server.sessionUsed = true server.sessionMu.Unlock() defer func() { - if !processStarted { + if command == nil { server.sessionMu.Lock() server.sessionUsed = false server.sessionMu.Unlock() @@ -174,13 +174,12 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo http.Error(w, "Maximum clients reached", http.StatusServiceUnavailable) return } - var command *exec.Cmd var err error defer func() { server.connections.Remove(id) if command != nil && server.onSessionEnd != nil { var startErr error - if !processStarted { + if command.Process == nil { startErr = err } server.onSessionEnd(startErr) @@ -192,7 +191,6 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo command = server.createServerCommand(ctx) return command }, w, r) - processStarted = command != nil && command.Process != nil if err != nil { log.Errorf(ctx, "Proxy server error: %v", err) } else { diff --git a/experimental/ssh/internal/proxy/server_options_test.go b/experimental/ssh/internal/proxy/server_options_test.go index cb0357ea078..c4fc284c3d4 100644 --- a/experimental/ssh/internal/proxy/server_options_test.go +++ b/experimental/ssh/internal/proxy/server_options_test.go @@ -2,11 +2,14 @@ package proxy import ( "context" + "fmt" "io" "net/http" "net/http/httptest" "os/exec" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -14,7 +17,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestSingleSessionReportsStartupFailureAndAllowsRetry(t *testing.T) { +func TestSingleSessionReportsStartupFailureAndRejectsRetry(t *testing.T) { for _, tc := range []struct{ stage, errorText string }{ {"stdin", "failed to get stdin pipe"}, {"stdout", "failed to get stdout pipe"}, @@ -33,33 +36,72 @@ func TestSingleSessionReportsStartupFailureAndAllowsRetry(t *testing.T) { } return cmd }, WithSingleSession(), WithSessionCompletion(func(err error) { completed <- err })) - handled := make(chan struct{}, 1) + handled := make(chan struct{}, 16) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { handler.ServeHTTP(w, r) handled <- struct{}{} })) defer server.Close() - for range 2 { - conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), "ws"+server.URL[4:]+"/ssh?id=retry", nil) - if response != nil { - response.Body.Close() + endpoint := "ws" + server.URL[4:] + "/ssh?id=retry" + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), endpoint, nil) + if response != nil { + response.Body.Close() + } + require.NoError(t, err) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, _, err = conn.ReadMessage() + conn.Close() + require.Error(t, err) + select { + case <-handled: + case <-time.After(5 * time.Second): + t.Fatal("failed startup did not release its handler") + } + require.ErrorContains(t, <-completed, tc.errorText) + require.Zero(t, manager.Count()) + for _, id := range []string{"retry", "another-session"} { + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), "ws"+server.URL[4:]+"/ssh?id="+id, nil) + if conn != nil { + conn.Close() } - require.NoError(t, err, "startup failure must not consume the single session") - require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) - _, _, err = conn.ReadMessage() - conn.Close() require.Error(t, err) - select { - case <-handled: - case <-time.After(5 * time.Second): - t.Fatal("failed startup did not release its handler") - } - require.ErrorContains(t, <-completed, tc.errorText) - require.False(t, handler.sessionUsed) - require.Zero(t, manager.Count()) + require.NotNil(t, response) + require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + response.Body.Close() + } + }) + } +} + +func TestSingleSessionConcurrentStartupFailuresInvokeFactoryOnce(t *testing.T) { + manager := NewConnectionsManager(1, 0, WithoutIdleShutdown()) + var starts, completions atomic.Int32 + handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd { + starts.Add(1) + return exec.CommandContext(ctx, "missing-sshd-executable") + }, WithSingleSession(), WithSessionCompletion(func(error) { completions.Add(1) })) + server := httptest.NewServer(handler) + defer server.Close() + var clients sync.WaitGroup + for i := range 16 { + clients.Go(func() { + endpoint := fmt.Sprintf("ws%s/ssh?id=session-%d", server.URL[4:], i) + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), endpoint, nil) + if response != nil { + response.Body.Close() + } + if err == nil { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, _, _ = conn.ReadMessage() + conn.Close() } }) } + clients.Wait() + server.Close() + require.EqualValues(t, 1, starts.Load()) + require.EqualValues(t, 1, completions.Load()) + require.Zero(t, manager.Count()) } func TestManagerCanDisableIdleShutdown(t *testing.T) { diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index 5beb65ca54c..dbad9a0317f 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -18,6 +18,7 @@ import ( "runtime" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -140,15 +141,25 @@ func TestServerReportsSSHDStartFailureAfterReadiness(t *testing.T) { require.NoError(t, json.NewDecoder(reader).Decode(&ready)) // Force a real exec failure after the configuration check succeeded. require.NoError(t, os.Remove(sshd)) - conn, err := dialWebSocket(t.Context(), "ws://token:"+ready.Token+"@"+listener.Addr().String()+"/ssh?id=failed-start") - require.NoError(t, err) - defer conn.Close() + var clients sync.WaitGroup + for i := range 16 { + clients.Go(func() { + endpoint := fmt.Sprintf("ws://token:%s@%s/ssh?id=failed-start-%d", ready.Token, listener.Addr(), i) + conn, err := dialWebSocket(t.Context(), endpoint) + if err == nil { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, _, _ = conn.ReadMessage() + conn.Close() + } + }) + } select { case err := <-finished: require.ErrorContains(t, err, "failed to start SSHD process") case <-time.After(5 * time.Second): t.Fatal("sshd startup failure did not stop the server") } + clients.Wait() } func TestServerStopsAfterItsSessionEnds(t *testing.T) { From cd5c2165022ec908f438fdf9085d2b5a3636f0a0 Mon Sep 17 00:00:00 2001 From: Hsing Tang Date: Thu, 19 Nov 2026 22:15:02 +0000 Subject: [PATCH 7/7] Wait for complete PID files and WebSocket handlers in sandbox tests --- acceptance/cmd/sandbox/server/server_test.py | 5 ++++- .../ssh/internal/proxy/server_options_test.go | 13 ++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/acceptance/cmd/sandbox/server/server_test.py b/acceptance/cmd/sandbox/server/server_test.py index bfcd410a06f..aa6dc89d84e 100644 --- a/acceptance/cmd/sandbox/server/server_test.py +++ b/acceptance/cmd/sandbox/server/server_test.py @@ -23,7 +23,10 @@ sshd = directory / "sshd" # A long-running child exercises cancellation without requiring system sshd. sshd.write_text( - '#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\necho $$ > "' + str(child_pid) + '"\nexec sleep 300\n' + '#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\n' + f'echo $$ > "{child_pid}.tmp"\n' + f'mv "{child_pid}.tmp" "{child_pid}"\n' + "exec sleep 300\n" ) sshd.chmod(0o700) help_result = subprocess.run( diff --git a/experimental/ssh/internal/proxy/server_options_test.go b/experimental/ssh/internal/proxy/server_options_test.go index c4fc284c3d4..4c953a51f1d 100644 --- a/experimental/ssh/internal/proxy/server_options_test.go +++ b/experimental/ssh/internal/proxy/server_options_test.go @@ -80,7 +80,11 @@ func TestSingleSessionConcurrentStartupFailuresInvokeFactoryOnce(t *testing.T) { starts.Add(1) return exec.CommandContext(ctx, "missing-sshd-executable") }, WithSingleSession(), WithSessionCompletion(func(error) { completions.Add(1) })) - server := httptest.NewServer(handler) + handled := make(chan struct{}, 16) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler.ServeHTTP(w, r) + handled <- struct{}{} + })) defer server.Close() var clients sync.WaitGroup for i := range 16 { @@ -98,6 +102,13 @@ func TestSingleSessionConcurrentStartupFailuresInvokeFactoryOnce(t *testing.T) { }) } clients.Wait() + for range 16 { + select { + case <-handled: + case <-time.After(5 * time.Second): + t.Fatal("failed startup did not release its handler") + } + } server.Close() require.EqualValues(t, 1, starts.Load()) require.EqualValues(t, 1, completions.Load())