Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

# SSH tunnel
/experimental/ssh/ @databricks/eng-deco-cli @databricks/eng-deco-ax
/experimental/ssh/sandbox/ @databricks/eng-deco-cli @databricks/eng-sandbox
/acceptance/ssh/ @databricks/eng-deco-cli @databricks/eng-deco-ax

# Apps
Expand Down
3 changes: 3 additions & 0 deletions acceptance/cmd/sandbox/server/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
GOOS.darwin = false
GOOS.windows = false
5 changes: 5 additions & 0 deletions acceptance/cmd/sandbox/server/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

>>> python3 server_test.py
Server remains hidden from sandbox help
Server starts without workspace credentials
SIGTERM stops the server and its SSH child
1 change: 1 addition & 0 deletions acceptance/cmd/sandbox/server/script
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
trace python3 server_test.py
87 changes: 87 additions & 0 deletions acceptance/cmd/sandbox/server/server_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import base64
import json
import os
import re
import signal
import socket
import subprocess
import tempfile
import time
from pathlib import Path

cli = os.environ["CLI"]
environment = {
key: value
for key, value in os.environ.items()
if not key.startswith(("DATABRICKS_", "ARM_", "AZURE_", "AWS_", "GOOGLE_"))
}
with tempfile.TemporaryDirectory() as temporary:
directory = Path(temporary)
environment["HOME"] = temporary
environment["DATABRICKS_CONFIG_FILE"] = str(directory / "missing-config")
child_pid = directory / "child.pid"
sshd = directory / "sshd"
# A long-running child exercises cancellation without requiring system sshd.
sshd.write_text(
'#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\n'
f'echo $$ > "{child_pid}.tmp"\n'
f'mv "{child_pid}.tmp" "{child_pid}"\n'
"exec sleep 300\n"
)
sshd.chmod(0o700)
help_result = subprocess.run(
[cli, "sandbox", "--help"], env=environment, capture_output=True, text=True, check=True
)
assert not re.search(r"^\s+server\s", help_result.stdout, re.MULTILINE)
print("Server remains hidden from sandbox help")
process = subprocess.Popen(
[cli, "sandbox", "server", "--port", "0", "--sshd", str(sshd), "--sshd-config", "prepared-config"],
env=environment,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
start_new_session=True,
)
try:
ready = json.loads(process.stdout.readline())
assert 10000 <= ready["port"] <= 65535
assert ready["token"]
print("Server starts without workspace credentials")
with socket.create_connection(("127.0.0.1", ready["port"]), timeout=5) as connection:
key = base64.b64encode(os.urandom(16)).decode()
connection.sendall(
(
"GET /ssh?id=acceptance HTTP/1.1\r\nHost: localhost\r\n"
"Upgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\n"
f"Sec-WebSocket-Key: {key}\r\nX-Sandbox-SSH-Token: {ready['token']}\r\n\r\n"
).encode()
)
assert connection.recv(4096).startswith(b"HTTP/1.1 101")
deadline = time.monotonic() + 5
while not child_pid.exists():
assert time.monotonic() < deadline, "sshd child did not start"
time.sleep(0.01)
pid = int(child_pid.read_text())
Comment thread
HsingTang marked this conversation as resolved.
Resolved
os.kill(pid, 0)
assert process.poll() is None, "server exited before SIGTERM"
process.send_signal(signal.SIGTERM)
assert process.wait(timeout=10) == 1 # context cancellation, rather than signal termination
try:
os.kill(pid, 0)
except ProcessLookupError:
pass
else:
raise AssertionError("sshd child survived server shutdown")
print("SIGTERM stops the server and its SSH child")
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
3 changes: 3 additions & 0 deletions acceptance/cmd/sandbox/server/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# The server and its signal handling run inside Linux sandboxes.
GOOS.windows = false
GOOS.darwin = false
12 changes: 11 additions & 1 deletion cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
aitoolscmd "github.com/databricks/cli/cmd/aitools"
"github.com/databricks/cli/cmd/psql"
ssh "github.com/databricks/cli/experimental/ssh/cmd"
sandboxssh "github.com/databricks/cli/experimental/ssh/sandbox"

"github.com/databricks/cli/cmd/account"
"github.com/databricks/cli/cmd/api"
Expand Down Expand Up @@ -91,6 +92,15 @@ func New(ctx context.Context) *root.Tree {
return cli
}

func sandboxSpec() root.CommandSpec {
spec := sandbox.Spec
spec.Build = func(ctx context.Context, cmd *cobra.Command) {
sandbox.Spec.Build(ctx, cmd)
cmd.AddCommand(sandboxssh.NewServerCommand())
}
return spec
}

// topLevelSpecs returns the specs of the CLI's top-level commands.
func topLevelSpecs() []root.CommandSpec {
specs := []root.CommandSpec{
Expand All @@ -108,7 +118,7 @@ func topLevelSpecs() []root.CommandSpec {
configure.Spec,
fs.Spec,
labs.Spec,
sandbox.Spec,
sandboxSpec(),
sync.Spec,
version.Spec,
quickstart.Spec,
Expand Down
35 changes: 35 additions & 0 deletions experimental/ssh/internal/proxy/client_server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -562,3 +562,38 @@ func TestClientTimesOutWhenServerSendsNothing(t *testing.T) {
t.Fatal("RunClientProxy did not abort on the handshake timeout")
}
}

func TestSingleSessionCannotStartAgainAfterDisconnect(t *testing.T) {
manager := NewConnectionsManager(1, 0, WithoutIdleShutdown())
completed := make(chan struct{})
var started atomic.Int32
handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd {
started.Add(1)
return exec.CommandContext(ctx, "cat")
}, WithSingleSession(), WithSessionCompletion(func(error) { close(completed) }))
server := httptest.NewServer(handler)
defer server.Close()
connection, response, err := websocket.DefaultDialer.Dial("ws"+server.URL[4:]+"?id=session", nil)
if response != nil {
response.Body.Close()
}
require.NoError(t, err)
require.NoError(t, connection.WriteMessage(websocket.BinaryMessage, []byte("first-session")))
require.NoError(t, connection.SetReadDeadline(time.Now().Add(5*time.Second)))
_, message, err := connection.ReadMessage()
require.NoError(t, err)
require.Equal(t, "first-session", string(message))
require.NoError(t, connection.Close())
select {
case <-completed:
case <-time.After(5 * time.Second):
t.Fatal("logical session completion was not reported")
}
require.Zero(t, manager.Count())
for _, id := range []string{"session", "another-session"} {
response := httptest.NewRecorder()
handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id="+id, nil))
require.Equal(t, http.StatusServiceUnavailable, response.Code)
}
require.Equal(t, int32(1), started.Load())
}
31 changes: 23 additions & 8 deletions experimental/ssh/internal/proxy/connections.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,34 @@ import (
// ConnectionsManager manages concurrent websocket clients and sends a shutdown signal if no
// clients are connected for a specified duration.
type ConnectionsManager struct {
maxClients int
shutdownDelay time.Duration
shutdownTimer *time.Timer
shutdownTimerMu sync.Mutex
connections map[string]*proxyConnection
connectionsMu sync.Mutex
TimedOut chan bool
maxClients int
shutdownDelay time.Duration
shutdownTimer *time.Timer
shutdownTimerMu sync.Mutex
disableIdleShutdown bool
connections map[string]*proxyConnection
connectionsMu sync.Mutex
TimedOut chan bool
}

func NewConnectionsManager(maxClients int, shutdownDelay time.Duration) *ConnectionsManager {
// ConnectionsManagerOption configures connection lifetime management.
type ConnectionsManagerOption func(*ConnectionsManager)

// WithoutIdleShutdown leaves server lifetime management to the caller.
func WithoutIdleShutdown() ConnectionsManagerOption {
return func(cm *ConnectionsManager) { cm.disableIdleShutdown = true }
}

func NewConnectionsManager(maxClients int, shutdownDelay time.Duration, options ...ConnectionsManagerOption) *ConnectionsManager {
cm := &ConnectionsManager{
maxClients: maxClients,
shutdownDelay: shutdownDelay,
connections: make(map[string]*proxyConnection),
TimedOut: make(chan bool),
}
for _, option := range options {
option(cm)
}
cm.startShutdownTimer(shutdownDelay)
return cm
}
Expand Down Expand Up @@ -80,6 +92,9 @@ func (cm *ConnectionsManager) ExtendIdleTimeout(delay time.Duration) {
}

func (cm *ConnectionsManager) startShutdownTimer(delay time.Duration) {
if cm.disableIdleShutdown {
return
}
cm.shutdownTimerMu.Lock()
defer cm.shutdownTimerMu.Unlock()
if cm.shutdownTimer != nil {
Expand Down
66 changes: 62 additions & 4 deletions experimental/ssh/internal/proxy/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os"
"os/exec"
"strconv"
"sync"
"time"

"github.com/databricks/cli/libs/log"
Expand All @@ -23,15 +24,37 @@ type proxyServer struct {
connections *ConnectionsManager
createServerCommand createServerCommandFunc
resumeGrace time.Duration
onSessionEnd func(error)
singleSession bool
sessionUsed bool
sessionMu sync.Mutex
}

func NewProxyServer(ctx context.Context, connections *ConnectionsManager, createServerCommand createServerCommandFunc) *proxyServer {
return &proxyServer{
// ServerOption configures the SSH transport capabilities.
type ServerOption func(*proxyServer)

// WithSessionCompletion reports process startup failures or a completed logical session.
// Established sessions report nil; handovers and failed upgrades do not notify.
func WithSessionCompletion(notify func(error)) ServerOption {
return func(server *proxyServer) { server.onSessionEnd = notify }
}

// WithSingleSession permits handovers but never starts a second SSH session.
func WithSingleSession() ServerOption {
return func(server *proxyServer) { server.singleSession = true }
}

func NewProxyServer(ctx context.Context, connections *ConnectionsManager, createServerCommand createServerCommandFunc, options ...ServerOption) *proxyServer {
server := &proxyServer{
ctx: ctx,
connections: connections,
createServerCommand: createServerCommand,
resumeGrace: proxyResumeGrace,
}
for _, option := range options {
option(server)
}
return server
}

func (server *proxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -117,6 +140,26 @@ func (server *proxyServer) handleExistingConnection(ctx context.Context, w http.
}

func (server *proxyServer) handleNewConnection(ctx context.Context, w http.ResponseWriter, r *http.Request, id string, req DialRequest) {
// Failed upgrades can retry; invoking the process factory consumes this server.
var command *exec.Cmd
if server.singleSession {
server.sessionMu.Lock()
if server.sessionUsed {
server.sessionMu.Unlock()
http.Error(w, "SSH session already used", http.StatusServiceUnavailable)
return
}
server.sessionUsed = true
server.sessionMu.Unlock()
defer func() {
if command == nil {
server.sessionMu.Lock()
server.sessionUsed = false
server.sessionMu.Unlock()
}
}()
}

// The server never dials, so it passes no connection factory: reattaching, for it, means
// waiting for the client to come back.
var conn *proxyConnection
Expand All @@ -131,10 +174,23 @@ func (server *proxyServer) handleNewConnection(ctx context.Context, w http.Respo
http.Error(w, "Maximum clients reached", http.StatusServiceUnavailable)
return
}
defer server.connections.Remove(id)
var err error
defer func() {
server.connections.Remove(id)
if command != nil && server.onSessionEnd != nil {
var startErr error
if command.Process == nil {
startErr = err
}
server.onSessionEnd(startErr)
}
}()

log.Infof(ctx, "Starting proxy server for new connection, count: %d", server.connections.Count())
err := runServerProxy(ctx, conn, server.createServerCommand, w, r)
err = runServerProxy(ctx, conn, func(ctx context.Context) *exec.Cmd {
command = server.createServerCommand(ctx)
return command
}, w, r)
if err != nil {
log.Errorf(ctx, "Proxy server error: %v", err)
} else {
Expand All @@ -161,11 +217,13 @@ func runServerProxy(ctx context.Context, proxy *proxyConnection, createServerCom
if err != nil {
return fmt.Errorf("failed to get stdin pipe: %w", err)
}
defer sshdStdin.Close()

sshdStdout, err := serverCmd.StdoutPipe()
if err != nil {
return fmt.Errorf("failed to get stdout pipe: %w", err)
}
defer sshdStdout.Close()

err = serverCmd.Start()
if err != nil {
Expand Down
Loading
Loading