agent: fix Connect CA data race and cache watcher timer leak

- leader_connect_ca.go: Hold stateLock across primaryRoots.Roots iteration in secondaryGetActivePrimaryCARoot() to avoid data race.
- watch.go: Replace time.After with time.NewTimer and timer.Stop() in notifyBlockingQuery and notifyPollingQuery to prevent memory leaks on context cancellation.

Author: mandar1045

.changelog/23854.txt

@@ -0,0 +1,4 @@
+```release-note:bug
+connect: Fix data race in secondaryGetActivePrimaryCARoot by holding stateLock during primary root slice iteration.
+agent/cache: Fix memory leak in notifyBlockingQuery and notifyPollingQuery timer loops by replacing time.After with time.NewTimer and explicit timer.Stop().
+```

agent/cache/watch.go

@@ -141,9 +141,11 @@ func (c *Cache) notifyBlockingQuery(ctx context.Context, r getOptions, correlati
 		}
 
 		if wait > 0 {
+			timer := time.NewTimer(wait)
 			select {
-			case <-time.After(wait):
+			case <-timer.C:
 			case <-ctx.Done():
+				timer.Stop()
 				return
 			}
 		}
@@ -239,10 +241,20 @@ func (c *Cache) notifyPollingQuery(ctx context.Context, r getOptions, correlatio
 			wait += lib.RandomStagger(r.Info.MaxAge / 16)
 		}
 
-		select {
-		case <-time.After(wait):
-		case <-ctx.Done():
-			return
+		if wait > 0 {
+			timer := time.NewTimer(wait)
+			select {
+			case <-timer.C:
+			case <-ctx.Done():
+				timer.Stop()
+				return
+			}
+		} else {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
 		}
 	}
 }

agent/consul/leader_connect_ca.go

@@ -203,13 +203,11 @@ func (c *CAManager) secondarySetPrimaryRoots(newRoots structs.IndexedCARoots) {
 }
 
 func (c *CAManager) secondaryGetActivePrimaryCARoot() (*structs.CARoot, error) {
-	// TODO: this could be a different lock, as long as its the same lock in secondarySetPrimaryRoots
 	c.stateLock.Lock()
-	primaryRoots := c.primaryRoots
-	c.stateLock.Unlock()
+	defer c.stateLock.Unlock()
 
-	for _, root := range primaryRoots.Roots {
-		if root.ID == primaryRoots.ActiveRootID && root.Active {
+	for _, root := range c.primaryRoots.Roots {
+		if root.ID == c.primaryRoots.ActiveRootID && root.Active {
 			return root, nil
 		}
 	}