Skip to content

Commit a6b20fd

Browse files
committed
Add internal sandbox WebSocket SSH server
1 parent e09a58e commit a6b20fd

16 files changed

Lines changed: 969 additions & 14 deletions

File tree

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
# SSH tunnel
2828
/experimental/ssh/ @databricks/eng-deco-cli @databricks/eng-deco-ax
29+
/experimental/ssh/sandbox/ @databricks/eng-deco-cli @databricks/eng-sandbox
2930
/acceptance/ssh/ @databricks/eng-deco-cli @databricks/eng-deco-ax
3031

3132
# Apps
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Cloud = false
2+
GOOS.darwin = false
3+
GOOS.windows = false
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
2+
>>> python3 server_test.py
3+
Server remains hidden from sandbox help
4+
Server starts without workspace credentials
5+
SIGTERM stops the server and its SSH child
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
trace python3 server_test.py
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import base64
2+
import json
3+
import os
4+
import re
5+
import signal
6+
import socket
7+
import subprocess
8+
import tempfile
9+
import time
10+
from pathlib import Path
11+
12+
cli = os.environ["CLI"]
13+
environment = {
14+
key: value
15+
for key, value in os.environ.items()
16+
if not key.startswith(("DATABRICKS_", "ARM_", "AZURE_", "AWS_", "GOOGLE_"))
17+
}
18+
with tempfile.TemporaryDirectory() as temporary:
19+
directory = Path(temporary)
20+
environment["HOME"] = temporary
21+
environment["DATABRICKS_CONFIG_FILE"] = str(directory / "missing-config")
22+
child_pid = directory / "child.pid"
23+
sshd = directory / "sshd"
24+
# A long-running child exercises cancellation without requiring system sshd.
25+
sshd.write_text('#!/bin/sh\nif [ "$1" = "-t" ]; then exit 0; fi\necho $$ > "' + str(child_pid) + '"\nexec cat\n')
26+
sshd.chmod(0o700)
27+
help_result = subprocess.run(
28+
[cli, "sandbox", "--help"], env=environment, capture_output=True, text=True, check=True
29+
)
30+
assert not re.search(r"^\s+server\s", help_result.stdout, re.MULTILINE)
31+
print("Server remains hidden from sandbox help")
32+
process = subprocess.Popen(
33+
[cli, "sandbox", "server", "--port", "0", "--sshd", str(sshd), "--sshd-config", "prepared-config"],
34+
env=environment,
35+
stdout=subprocess.PIPE,
36+
stderr=subprocess.PIPE,
37+
text=True,
38+
)
39+
try:
40+
ready = json.loads(process.stdout.readline())
41+
assert 10000 <= ready["port"] <= 65535
42+
assert ready["token"]
43+
print("Server starts without workspace credentials")
44+
with socket.create_connection(("127.0.0.1", ready["port"]), timeout=5) as connection:
45+
key = base64.b64encode(os.urandom(16)).decode()
46+
connection.sendall(
47+
(
48+
"GET /ssh?id=acceptance HTTP/1.1\r\nHost: localhost\r\n"
49+
"Upgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\n"
50+
f"Sec-WebSocket-Key: {key}\r\nX-Sandbox-SSH-Token: {ready['token']}\r\n\r\n"
51+
).encode()
52+
)
53+
assert connection.recv(4096).startswith(b"HTTP/1.1 101")
54+
deadline = time.monotonic() + 5
55+
while not child_pid.exists():
56+
assert time.monotonic() < deadline, "sshd child did not start"
57+
time.sleep(0.01)
58+
pid = int(child_pid.read_text())
59+
process.send_signal(signal.SIGTERM)
60+
assert process.wait(timeout=10) == 1 # context cancellation, rather than signal termination
61+
try:
62+
os.kill(pid, 0)
63+
except ProcessLookupError:
64+
pass
65+
else:
66+
raise AssertionError("sshd child survived server shutdown")
67+
print("SIGTERM stops the server and its SSH child")
68+
finally:
69+
if process.poll() is None:
70+
process.kill()
71+
process.wait()
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# The server and its signal handling run inside Linux sandboxes.
2+
GOOS.windows = false
3+
GOOS.darwin = false
4+
Ignore = [".databricks", ".ssh", "empty-home", "child.pid"]

‎cmd/cmd.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
aitoolscmd "github.com/databricks/cli/cmd/aitools"
99
"github.com/databricks/cli/cmd/psql"
1010
ssh "github.com/databricks/cli/experimental/ssh/cmd"
11+
sandboxssh "github.com/databricks/cli/experimental/ssh/sandbox"
1112

1213
"github.com/databricks/cli/cmd/account"
1314
"github.com/databricks/cli/cmd/api"
@@ -91,6 +92,15 @@ func New(ctx context.Context) *root.Tree {
9192
return cli
9293
}
9394

95+
func sandboxSpec() root.CommandSpec {
96+
spec := sandbox.Spec
97+
spec.Build = func(ctx context.Context, cmd *cobra.Command) {
98+
sandbox.Spec.Build(ctx, cmd)
99+
cmd.AddCommand(sandboxssh.NewServerCommand())
100+
}
101+
return spec
102+
}
103+
94104
// topLevelSpecs returns the specs of the CLI's top-level commands.
95105
func topLevelSpecs() []root.CommandSpec {
96106
specs := []root.CommandSpec{
@@ -108,7 +118,7 @@ func topLevelSpecs() []root.CommandSpec {
108118
configure.Spec,
109119
fs.Spec,
110120
labs.Spec,
111-
sandbox.Spec,
121+
sandboxSpec(),
112122
sync.Spec,
113123
version.Spec,
114124
quickstart.Spec,

‎experimental/ssh/internal/proxy/client_server_test.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,3 +562,33 @@ func TestClientTimesOutWhenServerSendsNothing(t *testing.T) {
562562
t.Fatal("RunClientProxy did not abort on the handshake timeout")
563563
}
564564
}
565+
566+
func TestSingleSessionCannotStartAgainAfterDisconnect(t *testing.T) {
567+
manager := NewConnectionsManager(1, 0, WithoutIdleShutdown())
568+
defer manager.Close()
569+
var started atomic.Int32
570+
handler := NewProxyServer(t.Context(), manager, func(ctx context.Context) *exec.Cmd {
571+
started.Add(1)
572+
return exec.CommandContext(ctx, "cat")
573+
}, WithSingleSession())
574+
server := httptest.NewServer(handler)
575+
defer server.Close()
576+
connection, response, err := websocket.DefaultDialer.Dial("ws"+server.URL[4:]+"?id=session", nil)
577+
if response != nil {
578+
response.Body.Close()
579+
}
580+
require.NoError(t, err)
581+
require.NoError(t, connection.WriteMessage(websocket.BinaryMessage, []byte("first-session")))
582+
require.NoError(t, connection.SetReadDeadline(time.Now().Add(5*time.Second)))
583+
_, message, err := connection.ReadMessage()
584+
require.NoError(t, err)
585+
require.Equal(t, "first-session", string(message))
586+
require.NoError(t, connection.Close())
587+
require.Eventually(t, func() bool { return manager.Count() == 0 }, 5*time.Second, time.Millisecond)
588+
for _, id := range []string{"session", "another-session"} {
589+
response := httptest.NewRecorder()
590+
handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/ssh?id="+id, nil))
591+
require.Equal(t, http.StatusServiceUnavailable, response.Code)
592+
}
593+
require.Equal(t, int32(1), started.Load())
594+
}

‎experimental/ssh/internal/proxy/connections.go‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,33 @@ import (
88
// ConnectionsManager manages concurrent websocket clients and sends a shutdown signal if no
99
// clients are connected for a specified duration.
1010
type ConnectionsManager struct {
11-
maxClients int
12-
shutdownDelay time.Duration
13-
shutdownTimer *time.Timer
14-
shutdownTimerMu sync.Mutex
15-
connections map[string]*proxyConnection
16-
connectionsMu sync.Mutex
17-
TimedOut chan bool
11+
maxClients int
12+
shutdownDelay time.Duration
13+
shutdownTimer *time.Timer
14+
shutdownTimerMu sync.Mutex
15+
disableIdleShutdown bool
16+
connections map[string]*proxyConnection
17+
connectionsMu sync.Mutex
18+
TimedOut chan bool
1819
}
1920

20-
func NewConnectionsManager(maxClients int, shutdownDelay time.Duration) *ConnectionsManager {
21+
// ConnectionsManagerOption configures connection lifetime management.
22+
type ConnectionsManagerOption func(*ConnectionsManager)
23+
24+
// WithoutIdleShutdown leaves server lifetime management to the caller.
25+
func WithoutIdleShutdown() ConnectionsManagerOption {
26+
return func(cm *ConnectionsManager) { cm.disableIdleShutdown = true }
27+
}
28+
29+
func NewConnectionsManager(maxClients int, shutdownDelay time.Duration, options ...ConnectionsManagerOption) *ConnectionsManager {
2130
cm := &ConnectionsManager{
2231
maxClients: maxClients,
2332
shutdownDelay: shutdownDelay,
2433
connections: make(map[string]*proxyConnection),
25-
TimedOut: make(chan bool),
34+
TimedOut: make(chan bool, 1),
35+
}
36+
for _, option := range options {
37+
option(cm)
2638
}
2739
cm.startShutdownTimer(shutdownDelay)
2840
return cm
@@ -80,13 +92,19 @@ func (cm *ConnectionsManager) ExtendIdleTimeout(delay time.Duration) {
8092
}
8193

8294
func (cm *ConnectionsManager) startShutdownTimer(delay time.Duration) {
95+
if cm.disableIdleShutdown {
96+
return
97+
}
8398
cm.shutdownTimerMu.Lock()
8499
defer cm.shutdownTimerMu.Unlock()
85100
if cm.shutdownTimer != nil {
86101
cm.shutdownTimer.Stop()
87102
}
88103
cm.shutdownTimer = time.AfterFunc(delay, func() {
89-
cm.TimedOut <- true
104+
select {
105+
case cm.TimedOut <- true:
106+
default:
107+
}
90108
})
91109
}
92110

@@ -97,3 +115,8 @@ func (cm *ConnectionsManager) cancelShutdownTimer() {
97115
cm.shutdownTimer.Stop()
98116
}
99117
}
118+
119+
// Close stops the idle timer.
120+
func (cm *ConnectionsManager) Close() {
121+
cm.cancelShutdownTimer()
122+
}

‎experimental/ssh/internal/proxy/connections_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,3 +239,16 @@ func TestConnectionsManager_ShutdownTimer_NoRestartWhenConnectionsRemain(t *test
239239
assert.False(t, timedOut, "Unexpected timeout signal while connections still exist")
240240
})
241241
}
242+
243+
func TestConnectionsManagerCloseStopsIdleTimer(t *testing.T) {
244+
synctest.Test(t, func(t *testing.T) {
245+
cm := NewConnectionsManager(1, time.Second)
246+
cm.Close()
247+
time.Sleep(2 * time.Second)
248+
select {
249+
case <-cm.TimedOut:
250+
t.Fatal("closed manager timed out")
251+
default:
252+
}
253+
})
254+
}

0 commit comments

Comments
 (0)