@@ -4,18 +4,27 @@ import (
44 "context"
55 "encoding/json"
66 "fmt"
7+ "net"
78 "os"
89 "os/exec"
910 "path/filepath"
1011 "runtime"
1112 "sort"
13+ "strconv"
1214 "strings"
1315 "sync"
16+ "time"
1417
1518 "github.com/NexusGPU/gpu-go/internal/errors"
1619 "github.com/NexusGPU/gpu-go/internal/platform"
1720)
1821
22+ var (
23+ createStabilityWindow = 3 * time .Second
24+ createStabilityPollInterval = 200 * time .Millisecond
25+ createSSHProbeTimeout = 500 * time .Millisecond
26+ )
27+
1928// Manager manages AI studio environments across different backends
2029type Manager struct {
2130 paths * platform.Paths
@@ -181,6 +190,12 @@ func (m *Manager) Create(ctx context.Context, opts *CreateOptions) (*Environment
181190 return nil , err
182191 }
183192
193+ if err := m .waitForStableRunning (ctx , backend , env ); err != nil {
194+ return nil , err
195+ }
196+
197+ m .clearUnreachableSSH (ctx , env )
198+
184199 // Save environment to local state
185200 if err := m .saveEnvironment (env ); err != nil {
186201 // Log but don't fail
@@ -345,6 +360,49 @@ func (m *Manager) Remove(ctx context.Context, idOrName string) error {
345360 return m .removeEnvironment (env .ID )
346361}
347362
363+ // RemoveAll removes all known environments. When runtimes are offline, stale state entries are still cleaned up.
364+ func (m * Manager ) RemoveAll (ctx context.Context ) ([]string , error ) {
365+ envs , err := m .List (ctx )
366+ if err != nil {
367+ return nil , err
368+ }
369+
370+ seen := make (map [string ]struct {}, len (envs ))
371+ removed := make ([]string , 0 , len (envs ))
372+ failed := make ([]string , 0 )
373+
374+ for _ , env := range envs {
375+ if env == nil || env .ID == "" {
376+ continue
377+ }
378+ if _ , ok := seen [env .ID ]; ok {
379+ continue
380+ }
381+ seen [env .ID ] = struct {}{}
382+
383+ switch env .Status {
384+ case StatusUnknown , StatusDeleted :
385+ if err := m .removeEnvironment (env .ID ); err != nil {
386+ failed = append (failed , fmt .Sprintf ("%s (%v)" , env .Name , err ))
387+ continue
388+ }
389+ removed = append (removed , env .Name )
390+ default :
391+ if err := m .Remove (ctx , env .ID ); err != nil {
392+ failed = append (failed , fmt .Sprintf ("%s (%v)" , env .Name , err ))
393+ continue
394+ }
395+ removed = append (removed , env .Name )
396+ }
397+ }
398+
399+ if len (failed ) > 0 {
400+ return removed , fmt .Errorf ("failed to remove %d environment(s): %s" , len (failed ), strings .Join (failed , "; " ))
401+ }
402+
403+ return removed , nil
404+ }
405+
348406// AddSSHConfig adds an SSH config entry for an environment
349407func (m * Manager ) AddSSHConfig (env * Environment ) error {
350408 if env .SSHHost == "" || env .SSHPort == 0 {
@@ -447,6 +505,109 @@ func (m *Manager) getSSHConfigPath() string {
447505 return filepath .Join (home , ".ssh" , "config" )
448506}
449507
508+ func (m * Manager ) waitForStableRunning (ctx context.Context , backend Backend , created * Environment ) error {
509+ if createStabilityWindow <= 0 {
510+ return nil
511+ }
512+ if created == nil || created .ID == "" {
513+ return fmt .Errorf ("studio creation returned invalid environment metadata" )
514+ }
515+
516+ deadline := time .Now ().Add (createStabilityWindow )
517+ seenRunning := false
518+ var lastStatus EnvironmentStatus
519+
520+ for {
521+ select {
522+ case <- ctx .Done ():
523+ return ctx .Err ()
524+ default :
525+ }
526+
527+ env , err := backend .Get (ctx , created .ID )
528+ if err != nil {
529+ return fmt .Errorf (
530+ "studio '%s' created but status check failed within %s: %w. Check status with `ggo studio list` and logs with `ggo studio logs %s`" ,
531+ created .Name ,
532+ createStabilityWindow ,
533+ err ,
534+ created .Name ,
535+ )
536+ }
537+ lastStatus = env .Status
538+
539+ switch env .Status {
540+ case StatusRunning :
541+ seenRunning = true
542+ case StatusPending , StatusStarting , StatusPulling :
543+ if seenRunning {
544+ return fmt .Errorf (
545+ "studio '%s' failed to stay running for at least %s (status=%s). Check status with `ggo studio list` and logs with `ggo studio logs %s`" ,
546+ created .Name ,
547+ createStabilityWindow ,
548+ env .Status ,
549+ created .Name ,
550+ )
551+ }
552+ default :
553+ return fmt .Errorf (
554+ "studio '%s' failed to stay running for at least %s (status=%s). Check status with `ggo studio list` and logs with `ggo studio logs %s`" ,
555+ created .Name ,
556+ createStabilityWindow ,
557+ env .Status ,
558+ created .Name ,
559+ )
560+ }
561+
562+ remaining := time .Until (deadline )
563+ if remaining <= 0 {
564+ if seenRunning {
565+ return nil
566+ }
567+ return fmt .Errorf (
568+ "studio '%s' did not reach running status within %s (last status=%s). Check status with `ggo studio list` and logs with `ggo studio logs %s`" ,
569+ created .Name ,
570+ createStabilityWindow ,
571+ lastStatus ,
572+ created .Name ,
573+ )
574+ }
575+
576+ sleepFor := createStabilityPollInterval
577+ if sleepFor <= 0 || sleepFor > remaining {
578+ sleepFor = remaining
579+ }
580+
581+ timer := time .NewTimer (sleepFor )
582+ select {
583+ case <- ctx .Done ():
584+ timer .Stop ()
585+ return ctx .Err ()
586+ case <- timer .C :
587+ }
588+ }
589+ }
590+
591+ func (m * Manager ) clearUnreachableSSH (ctx context.Context , env * Environment ) {
592+ if env == nil || env .SSHHost == "" || env .SSHPort <= 0 {
593+ return
594+ }
595+
596+ dialCtx , cancel := context .WithTimeout (ctx , createSSHProbeTimeout )
597+ defer cancel ()
598+
599+ dialer := & net.Dialer {}
600+ address := net .JoinHostPort (env .SSHHost , strconv .Itoa (env .SSHPort ))
601+ conn , err := dialer .DialContext (dialCtx , "tcp" , address )
602+ if err != nil {
603+ env .SSHHost = ""
604+ env .SSHPort = 0
605+ env .SSHUser = ""
606+ return
607+ }
608+ _ = conn .Close ()
609+ }
610+
450611// State management
451612
452613func (m * Manager ) getStatePath () string {
0 commit comments