Skip to content

Commit 79a42de

Browse files
andystimeclaude
andcommitted
feat: add comprehensive logging for connection info debugging
Added detailed logging to help diagnose connection info display issues: 1. Connection Directory Reading: - Log connections directory path (V5) - Log number of entries found in directory (V5) - Log when directory doesn't exist yet (V5) - Log each worker's active connections with details (V4) - Log total connections across all workers (V4) - Log when no connections are found (V5) 2. Connection Reporting: - Log when connections are reported to server (V4) - Added connection count to worker summary log - Changed summary format: restarts -> conns in output - Log in both hypervisor and config fallback modes 3. Worker Environment Setup: - Log TF_CONNECTION_INFO_PATH for each worker (V4) - Helps verify correct file path is set for tensor-fusion-worker These logs will help diagnose: - Whether connection files are being created by tensor-fusion-worker - Whether connection files are being read correctly by agent - Whether connections are being reported to the server - Path configuration issues Note: Connection files are populated by tensor-fusion-worker when clients connect. If no connections appear, the issue is likely in tensor-fusion-worker not writing to TF_CONNECTION_INFO_PATH. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 474a3e3 commit 79a42de

1 file changed

Lines changed: 27 additions & 3 deletions

File tree

internal/agent/agent.go

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,9 @@ func (a *Agent) convertToWorkerInfos(apiWorkers []api.WorkerConfig) ([]*hvApi.Wo
455455

456456
// Set TF_CONNECTION_INFO_PATH to the worker's specific connection file (not directory)
457457
// Worker will write connection info to this file, one line per connection
458-
envVars[EnvConnectionInfoPath] = filepath.Join(a.connectionsDir, w.WorkerID+".txt")
458+
connectionInfoPath := filepath.Join(a.connectionsDir, w.WorkerID+".txt")
459+
envVars[EnvConnectionInfoPath] = connectionInfoPath
460+
klog.V(4).Infof("Worker %s: Set %s=%s", w.WorkerID, EnvConnectionInfoPath, connectionInfoPath)
459461

460462
// Set hard limiter environment variables for Fractional GPU support
461463
// TODO: use MIG for partitioned
@@ -805,14 +807,19 @@ func (a *Agent) detectWorkerChanges(currentWorkers []*hvApi.WorkerInfo) map[stri
805807
func (a *Agent) readConnectionsFromDir() (map[string][]string, error) {
806808
connections := make(map[string][]string)
807809

810+
klog.V(5).Infof("Reading connections from directory: %s", a.connectionsDir)
811+
808812
entries, err := os.ReadDir(a.connectionsDir)
809813
if err != nil {
810814
if os.IsNotExist(err) {
815+
klog.V(5).Infof("Connections directory does not exist yet: %s", a.connectionsDir)
811816
return connections, nil // No connections directory yet
812817
}
813818
return nil, fmt.Errorf("failed to read connections directory: %w", err)
814819
}
815820

821+
klog.V(5).Infof("Found %d entries in connections directory", len(entries))
822+
816823
for _, entry := range entries {
817824
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".txt") {
818825
continue
@@ -833,9 +840,20 @@ func (a *Agent) readConnectionsFromDir() (map[string][]string, error) {
833840

834841
if len(connLines) > 0 {
835842
connections[workerID] = connLines
843+
klog.V(4).Infof("Worker %s has %d active connection(s): %v", workerID, len(connLines), connLines)
836844
}
837845
}
838846

847+
if len(connections) > 0 {
848+
totalConns := 0
849+
for _, conns := range connections {
850+
totalConns += len(conns)
851+
}
852+
klog.V(4).Infof("Total connections across all workers: %d", totalConns)
853+
} else {
854+
klog.V(5).Infof("No active connections found in any worker connection files")
855+
}
856+
839857
return connections, nil
840858
}
841859

@@ -1156,6 +1174,9 @@ func (a *Agent) collectWorkerStatusFromHypervisor(
11561174
var connections []api.ConnectionInfo
11571175
if connLines, ok := currentConnections[w.WorkerUID]; ok {
11581176
connections = parseConnectionsToAPI(connLines)
1177+
if len(connections) > 0 {
1178+
klog.V(4).Infof("Worker %s: Reporting %d connection(s) to server", w.WorkerUID, len(connections))
1179+
}
11591180
}
11601181

11611182
gpuIndices := resolveWorkerGPUIndices(w.WorkerUID, nil, w.AllocatedDevices, gpuIndexByID)
@@ -1171,8 +1192,8 @@ func (a *Agent) collectWorkerStatusFromHypervisor(
11711192
ConnectionChanged: &connectionChanged,
11721193
GPUChanged: &gpuChanged,
11731194
})
1174-
summaryParts = append(summaryParts, fmt.Sprintf("%s(status=%s,pid=%d,restarts=%d,wc=%v,cc=%v,gc=%v)",
1175-
w.WorkerUID, status, pid, restarts, workerChanged, connectionChanged, gpuChanged))
1195+
summaryParts = append(summaryParts, fmt.Sprintf("%s(status=%s,pid=%d,conns=%d,wc=%v,cc=%v,gc=%v)",
1196+
w.WorkerUID, status, pid, len(connections), workerChanged, connectionChanged, gpuChanged))
11761197
}
11771198

11781199
if len(hvWorkers) > 0 {
@@ -1208,6 +1229,9 @@ func (a *Agent) collectWorkerStatusFromConfig(
12081229
connections := w.Connections
12091230
if connLines, ok := currentConnections[w.WorkerID]; ok {
12101231
connections = parseConnectionsToAPI(connLines)
1232+
if len(connections) > 0 {
1233+
klog.V(4).Infof("Worker %s: Reporting %d connection(s) to server (config fallback)", w.WorkerID, len(connections))
1234+
}
12111235
}
12121236

12131237
gpuIndices := resolveWorkerGPUIndices(w.WorkerID, w.GPUIndices, w.GPUIDs, gpuIndexByID)

0 commit comments

Comments
 (0)