Skip to content

Commit c4a60d1

Browse files
committed
fix: enhance studio environment management with batch removal and stability checks
1 parent e127942 commit c4a60d1

7 files changed

Lines changed: 442 additions & 7 deletions

File tree

cmd/ggo/studio/studio.go

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,10 @@ Examples:
8181
ggo studio stop my-studio
8282
8383
# Remove an environment
84-
ggo studio rm my-studio`,
84+
ggo studio rm my-studio
85+
86+
# Batch remove all environments
87+
ggo studio rm --all -f`,
8588
PersistentPreRun: func(cmd *cobra.Command, args []string) {
8689
// Initialize klog flags if not already initialized
8790
klog.InitFlags(nil)
@@ -667,18 +670,61 @@ func newStopCmd() *cobra.Command {
667670

668671
func newRemoveCmd() *cobra.Command {
669672
var force bool
673+
var all bool
670674

671675
cmd := &cobra.Command{
672676
Use: "rm <name>",
673-
Short: "Remove a studio environment",
677+
Short: "Remove studio environment(s)",
674678
Aliases: []string{"remove", "delete"},
675-
Args: cobra.ExactArgs(1),
679+
Args: func(cmd *cobra.Command, args []string) error {
680+
if all {
681+
if len(args) > 0 {
682+
return fmt.Errorf("--all does not accept a studio name")
683+
}
684+
return nil
685+
}
686+
return cobra.ExactArgs(1)(cmd, args)
687+
},
676688
RunE: func(cmd *cobra.Command, args []string) error {
677689
ctx := context.Background()
678690
mgr := getManager()
679691
out := getOutput()
680-
name := args[0]
681692

693+
if all {
694+
if !force && !out.IsJSON() {
695+
styles := tui.DefaultStyles()
696+
fmt.Printf("%s Are you sure you want to remove ALL studio environments? [y/N]: ", styles.Warning.Render("!"))
697+
var confirm string
698+
fmt.Scanln(&confirm)
699+
if confirm != "y" && confirm != "Y" {
700+
out.Info("Cancelled")
701+
return nil
702+
}
703+
}
704+
705+
removedNames, err := mgr.RemoveAll(ctx)
706+
if err != nil {
707+
cmd.SilenceUsage = true
708+
return err
709+
}
710+
if len(removedNames) == 0 {
711+
out.Info("No studio environments found")
712+
return nil
713+
}
714+
for _, removedName := range removedNames {
715+
if err := mgr.RemoveSSHConfig(removedName); err != nil {
716+
klog.Warningf("Failed to remove SSH config for %s: error=%v", removedName, err)
717+
}
718+
}
719+
720+
return out.Render(&cmdutil.ActionData{
721+
Success: true,
722+
Message: fmt.Sprintf("Removed %d studio environment(s)", len(removedNames)),
723+
ID: "all",
724+
})
725+
}
726+
727+
name := args[0]
682728
if !force && !out.IsJSON() {
683729
styles := tui.DefaultStyles()
684730
fmt.Printf("%s Are you sure you want to remove environment %s? [y/N]: ",
@@ -710,6 +756,7 @@ func newRemoveCmd() *cobra.Command {
710756
}
711757

712758
cmd.Flags().BoolVarP(&force, "force", "f", false, "Force remove")
759+
cmd.Flags().BoolVar(&all, "all", false, "Remove all studio environments in one batch")
713760
return cmd
714761
}
715762

internal/studio/backend_apple.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ func extractSSHPort(published []appleContainerPublishedPort) int {
464464
}
465465
}
466466
}
467-
return 22
467+
return 0
468468
}
469469

470470
func normalizeContainerMemory(memory string) string {

internal/studio/backend_docker.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ func (b *DockerBackend) Get(ctx context.Context, idOrName string) (*Environment,
455455
}
456456

457457
// Parse SSH port
458-
sshPort := 22
458+
sshPort := 0
459459
if ports, ok := c.NetworkSettings.Ports["22/tcp"]; ok && len(ports) > 0 {
460460
if p, err := strconv.Atoi(ports[0].HostPort); err == nil {
461461
sshPort = p
@@ -562,7 +562,7 @@ func parseSSHPort(ports string) int {
562562
}
563563
}
564564
}
565-
return 22
565+
return 0
566566
}
567567

568568
// findAvailablePort finds an available port in the SSH port range (12000-18000)
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package studio
2+
3+
import (
4+
. "github.com/onsi/ginkgo/v2"
5+
. "github.com/onsi/gomega"
6+
)
7+
8+
var _ = Describe("SSH port parsing", func() {
9+
Describe("parseSSHPort", func() {
10+
It("returns mapped host SSH port when 22/tcp is published", func() {
11+
port := parseSSHPort("0.0.0.0:15432->22/tcp")
12+
Expect(port).To(Equal(15432))
13+
})
14+
15+
It("returns zero when no SSH port mapping exists", func() {
16+
Expect(parseSSHPort("0.0.0.0:8888->8888/tcp")).To(BeZero())
17+
Expect(parseSSHPort("")).To(BeZero())
18+
})
19+
})
20+
21+
Describe("extractSSHPort", func() {
22+
It("returns mapped host SSH port for Apple Container", func() {
23+
port := extractSSHPort([]appleContainerPublishedPort{{HostPort: 17000, ContainerPort: 22, Proto: "tcp", Count: 1}})
24+
Expect(port).To(Equal(17000))
25+
})
26+
27+
It("returns zero when Apple Container has no SSH mapping", func() {
28+
port := extractSSHPort([]appleContainerPublishedPort{{HostPort: 8080, ContainerPort: 8080, Proto: "tcp", Count: 1}})
29+
Expect(port).To(BeZero())
30+
})
31+
})
32+
})

internal/studio/manager.go

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,27 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"net"
78
"os"
89
"os/exec"
910
"path/filepath"
1011
"runtime"
1112
"sort"
13+
"strconv"
1214
"strings"
1315
"sync"
16+
"time"
1417

1518
"github.com/NexusGPU/gpu-go/internal/errors"
1619
"github.com/NexusGPU/gpu-go/internal/platform"
1720
)
1821

22+
var (
23+
createStabilityWindow = 3 * time.Second
24+
createStabilityPollInterval = 200 * time.Millisecond
25+
createSSHProbeTimeout = 500 * time.Millisecond
26+
)
27+
1928
// Manager manages AI studio environments across different backends
2029
type Manager struct {
2130
paths *platform.Paths
@@ -181,6 +190,12 @@ func (m *Manager) Create(ctx context.Context, opts *CreateOptions) (*Environment
181190
return nil, err
182191
}
183192

193+
if err := m.waitForStableRunning(ctx, backend, env); err != nil {
194+
return nil, err
195+
}
196+
197+
m.clearUnreachableSSH(ctx, env)
198+
184199
// Save environment to local state
185200
if err := m.saveEnvironment(env); err != nil {
186201
// Log but don't fail
@@ -345,6 +360,49 @@ func (m *Manager) Remove(ctx context.Context, idOrName string) error {
345360
return m.removeEnvironment(env.ID)
346361
}
347362

363+
// RemoveAll removes all known environments. When runtimes are offline, stale state entries are still cleaned up.
364+
func (m *Manager) RemoveAll(ctx context.Context) ([]string, error) {
365+
envs, err := m.List(ctx)
366+
if err != nil {
367+
return nil, err
368+
}
369+
370+
seen := make(map[string]struct{}, len(envs))
371+
removed := make([]string, 0, len(envs))
372+
failed := make([]string, 0)
373+
374+
for _, env := range envs {
375+
if env == nil || env.ID == "" {
376+
continue
377+
}
378+
if _, ok := seen[env.ID]; ok {
379+
continue
380+
}
381+
seen[env.ID] = struct{}{}
382+
383+
switch env.Status {
384+
case StatusUnknown, StatusDeleted:
385+
if err := m.removeEnvironment(env.ID); err != nil {
386+
failed = append(failed, fmt.Sprintf("%s (%v)", env.Name, err))
387+
continue
388+
}
389+
removed = append(removed, env.Name)
390+
default:
391+
if err := m.Remove(ctx, env.ID); err != nil {
392+
failed = append(failed, fmt.Sprintf("%s (%v)", env.Name, err))
393+
continue
394+
}
395+
removed = append(removed, env.Name)
396+
}
397+
}
398+
399+
if len(failed) > 0 {
400+
return removed, fmt.Errorf("failed to remove %d environment(s): %s", len(failed), strings.Join(failed, "; "))
401+
}
402+
403+
return removed, nil
404+
}
405+
348406
// AddSSHConfig adds an SSH config entry for an environment
349407
func (m *Manager) AddSSHConfig(env *Environment) error {
350408
if env.SSHHost == "" || env.SSHPort == 0 {
@@ -447,6 +505,109 @@ func (m *Manager) getSSHConfigPath() string {
447505
return filepath.Join(home, ".ssh", "config")
448506
}
449507

508+
func (m *Manager) waitForStableRunning(ctx context.Context, backend Backend, created *Environment) error {
509+
if createStabilityWindow <= 0 {
510+
return nil
511+
}
512+
if created == nil || created.ID == "" {
513+
return fmt.Errorf("studio creation returned invalid environment metadata")
514+
}
515+
516+
deadline := time.Now().Add(createStabilityWindow)
517+
seenRunning := false
518+
var lastStatus EnvironmentStatus
519+
520+
for {
521+
select {
522+
case <-ctx.Done():
523+
return ctx.Err()
524+
default:
525+
}
526+
527+
env, err := backend.Get(ctx, created.ID)
528+
if err != nil {
529+
return fmt.Errorf(
530+
"studio '%s' created but status check failed within %s: %w. Check status with `ggo studio list` and logs with `ggo studio logs %s`",
531+
created.Name,
532+
createStabilityWindow,
533+
err,
534+
created.Name,
535+
)
536+
}
537+
lastStatus = env.Status
538+
539+
switch env.Status {
540+
case StatusRunning:
541+
seenRunning = true
542+
case StatusPending, StatusStarting, StatusPulling:
543+
if seenRunning {
544+
return fmt.Errorf(
545+
"studio '%s' failed to stay running for at least %s (status=%s). Check status with `ggo studio list` and logs with `ggo studio logs %s`",
546+
created.Name,
547+
createStabilityWindow,
548+
env.Status,
549+
created.Name,
550+
)
551+
}
552+
default:
553+
return fmt.Errorf(
554+
"studio '%s' failed to stay running for at least %s (status=%s). Check status with `ggo studio list` and logs with `ggo studio logs %s`",
555+
created.Name,
556+
createStabilityWindow,
557+
env.Status,
558+
created.Name,
559+
)
560+
}
561+
562+
remaining := time.Until(deadline)
563+
if remaining <= 0 {
564+
if seenRunning {
565+
return nil
566+
}
567+
return fmt.Errorf(
568+
"studio '%s' did not reach running status within %s (last status=%s). Check status with `ggo studio list` and logs with `ggo studio logs %s`",
569+
created.Name,
570+
createStabilityWindow,
571+
lastStatus,
572+
created.Name,
573+
)
574+
}
575+
576+
sleepFor := createStabilityPollInterval
577+
if sleepFor <= 0 || sleepFor > remaining {
578+
sleepFor = remaining
579+
}
580+
581+
timer := time.NewTimer(sleepFor)
582+
select {
583+
case <-ctx.Done():
584+
timer.Stop()
585+
return ctx.Err()
586+
case <-timer.C:
587+
}
588+
}
589+
}
590+
591+
func (m *Manager) clearUnreachableSSH(ctx context.Context, env *Environment) {
592+
if env == nil || env.SSHHost == "" || env.SSHPort <= 0 {
593+
return
594+
}
595+
596+
dialCtx, cancel := context.WithTimeout(ctx, createSSHProbeTimeout)
597+
defer cancel()
598+
599+
dialer := &net.Dialer{}
600+
address := net.JoinHostPort(env.SSHHost, strconv.Itoa(env.SSHPort))
601+
conn, err := dialer.DialContext(dialCtx, "tcp", address)
602+
if err != nil {
603+
env.SSHHost = ""
604+
env.SSHPort = 0
605+
env.SSHUser = ""
606+
return
607+
}
608+
_ = conn.Close()
609+
}
610+
450611
// State management
451612

452613
func (m *Manager) getStatePath() string {

0 commit comments

Comments
 (0)