@@ -93,19 +93,56 @@ func StartStandaloneCacheServer(ctx context.Context, t *testing.T, dataDir strin
9393 cacheServerCompletedConfig , err := cacheServerConfig .Complete ()
9494 require .NoError (t , err )
9595
96+ // When the test ends the cache server is stopped.
97+ // For the cache server to stop gracefully etcd must be available until it is shut down.
98+ // For etcd to then gracefully stop the files on disk must be present until etcd is shut down.
99+ // If the cache server/etcd are bounded by the context the server
100+ // will shut down in parallel while the test harness is deleting
101+ // files, which leads to spurious opaque test errors (sometimes in
102+ // _other_ tests).
103+ // So the shutdown _must_ be:
104+ // 1. Stop cache server
105+ // 2. Stop etcd
106+ // 3. Let test cleanup run
107+ serverCtx , serverCancel := context .WithCancel (ctx )
108+ etcdCtx , etcdCancel := context .WithCancel (context .Background ())
109+
96110 if cacheServerCompletedConfig .EmbeddedEtcd .Config != nil {
97111 t .Logf ("Starting embedded etcd for the cache server" )
98- require .NoError (t , embeddedetcd .NewServer (cacheServerCompletedConfig .EmbeddedEtcd ).Run (ctx ))
112+
113+ etcdserver := embeddedetcd .NewServer (cacheServerCompletedConfig .EmbeddedEtcd )
114+
115+ if err := etcdserver .Run (etcdCtx ); err != nil {
116+ etcdCancel ()
117+ t .Fatalf ("failed to start embedded etcd: %v" , err )
118+ }
119+
120+ t .Cleanup (func () {
121+ <- etcdCtx .Done ()
122+ if err := wait .PollUntilContextTimeout (context .Background (), time .Second , wait .ForeverTestTimeout , true ,
123+ func (ctx context.Context ) (bool , error ) {
124+ return etcdserver .Stopped (), nil
125+ },
126+ ); err != nil {
127+ t .Fatal (err )
128+ }
129+ })
99130 }
131+
100132 cacheServer , err := cacheserver .NewServer (cacheServerCompletedConfig )
101133 require .NoError (t , err )
102- preparedCachedServer , err := cacheServer .PrepareRun (ctx )
134+ preparedCachedServer , err := cacheServer .PrepareRun (serverCtx )
103135 require .NoError (t , err )
104136 start := time .Now ()
137+
105138 t .Logf ("Starting the cache server" )
106139 go func () {
107- assert .NoError (t , preparedCachedServer .Run (ctx ))
140+ // stop etcd server after cache server is done shutting down
141+ defer etcdCancel ()
142+ assert .NoError (t , preparedCachedServer .Run (serverCtx ))
108143 }()
144+ // Stop cache server before starting other cleanup functions
145+ t .Cleanup (serverCancel )
109146
110147 cacheServerCertificatePath := path .Join (dataDir , "cache" , "apiserver.crt" )
111148 kcptestinghelpers .Eventually (t , func () (bool , string ) {
0 commit comments