Skip to content

Commit fa43883

Browse files
andystimeclaude
andcommitted
fix: improve studio reliability and agent heartbeat
- Fix environment variable: TF_CLIENT_INFO_PATH -> TF_CONNECTION_INFO_PATH to match tensor-fusion-worker expectations - Add TF_CONNECTION_INFO_PATH to studio containers with connections directory mount at /var/run/tensor-fusion/connections to enable worker connection tracking - Add default "sleep infinity" command for studio containers to prevent immediate exit when no command is specified - Reduce agent heartbeat interval from 60s to 30s for faster dashboard updates and more responsive worker status reporting These changes fix the "studio failed to stay running" issue and improve real-time status synchronization between ggo and the web dashboard. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent da2ab02 commit fa43883

4 files changed

Lines changed: 34 additions & 7 deletions

File tree

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ require (
152152
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
153153
github.com/spf13/pflag v1.0.10 // indirect
154154
golang.org/x/net v0.50.0 // indirect
155-
golang.org/x/sys v0.41.0 // indirect
155+
golang.org/x/sys v0.41.0
156156
golang.org/x/term v0.40.0
157157
golang.org/x/time v0.14.0 // indirect
158158
gopkg.in/yaml.v3 v3.0.1 // indirect

internal/agent/agent.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ import (
2323
)
2424

2525
const (
26-
statusReportInterval = 1 * time.Minute
26+
statusReportInterval = 30 * time.Second
2727
forceRefreshInterval = 6 * time.Hour
2828
// EnvConnectionInfoPath is the environment variable name for connection info directory path
29-
// Workers should write their connections to: {TF_CLIENT_INFO_PATH}/{workerID}.txt
30-
EnvConnectionInfoPath = "TF_CLIENT_INFO_PATH"
29+
// Workers should write their connections to: {TF_CONNECTION_INFO_PATH}/{workerID}.txt
30+
EnvConnectionInfoPath = "TF_CONNECTION_INFO_PATH"
3131

3232
// Worker status constants
3333
workerStatusRunning = "running"
@@ -230,8 +230,8 @@ func (a *Agent) Start() error {
230230
klog.Warningf("Failed to create connections directory: path=%s error=%v", a.connectionsDir, err)
231231
}
232232

233-
// Set TF_CLIENT_INFO_PATH environment variable for worker processes
234-
// Workers will write connection info to: {TF_CLIENT_INFO_PATH}/{workerID}.txt
233+
// Set TF_CONNECTION_INFO_PATH environment variable for worker processes
234+
// Workers will write connection info to: {TF_CONNECTION_INFO_PATH}/{workerID}.txt
235235
// Each worker has its own file with format: clientIP,clientPort,clientPID (one per line)
236236
if err := os.Setenv(EnvConnectionInfoPath, a.connectionsDir); err != nil {
237237
klog.Warningf("Failed to set %s env var: error=%v", EnvConnectionInfoPath, err)

internal/studio/backend_docker.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,12 @@ func (b *DockerBackend) Create(ctx context.Context, opts *CreateOptions) (*Envir
264264

265265
// Add command args (supplements ENTRYPOINT or overrides CMD)
266266
// FormatContainerCommand handles wrapping single shell commands with "sh -c"
267-
if formattedCmd := FormatContainerCommand(opts.Command); len(formattedCmd) > 0 {
267+
// If no command is provided, use "sleep infinity" to keep container running
268+
cmdToUse := opts.Command
269+
if len(cmdToUse) == 0 {
270+
cmdToUse = []string{"sleep", "infinity"}
271+
}
272+
if formattedCmd := FormatContainerCommand(cmdToUse); len(formattedCmd) > 0 {
268273
args = append(args, formattedCmd...)
269274
}
270275

internal/studio/env.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,21 @@ func SetupGPUEnv(paths *platform.Paths, config *GPUEnvConfig) (*GPUEnvResult, er
200200
result.EnvVars["TF_LOG_LEVEL"] = getEnvDefault("TF_LOG_LEVEL", "info")
201201
result.EnvVars["TF_ENABLE_LOG"] = getEnvDefault("TF_ENABLE_LOG", "1")
202202

203+
// Get connections directory (for tensor-fusion-worker to write connection info)
204+
connectionsDir := filepath.Join(paths.StateDir(), "connections")
205+
if err := os.MkdirAll(connectionsDir, 0755); err != nil {
206+
return nil, fmt.Errorf("failed to create connections directory: %w", err)
207+
}
208+
209+
// Set TF_CONNECTION_INFO_PATH for tensor-fusion-worker
210+
if !config.IsContainer {
211+
// On host, use actual connections directory
212+
result.EnvVars["TF_CONNECTION_INFO_PATH"] = connectionsDir
213+
} else {
214+
// In container, use mounted path
215+
result.EnvVars["TF_CONNECTION_INFO_PATH"] = "/var/run/tensor-fusion/connections"
216+
}
217+
203218
// Add cache path to PATH (for tensor-fusion-worker binary)
204219
if !config.IsContainer {
205220
// On host, prepend cache path to existing PATH
@@ -236,6 +251,13 @@ func SetupGPUEnv(paths *platform.Paths, config *GPUEnvConfig) (*GPUEnvResult, er
236251

237252
// Set up volume mounts for container mode
238253
if config.IsContainer {
254+
// Mount connections directory (for tensor-fusion-worker to write connection info)
255+
result.VolumeMounts = append(result.VolumeMounts, VolumeMount{
256+
HostPath: connectionsDir,
257+
ContainerPath: "/var/run/tensor-fusion/connections",
258+
ReadOnly: false,
259+
})
260+
239261
// Mount libs directory (contains only .so files for LD_LIBRARY_PATH/LD_PRELOAD)
240262
result.VolumeMounts = append(result.VolumeMounts, VolumeMount{
241263
HostPath: libsPath,

0 commit comments

Comments
 (0)