fix(rerank): prevent Darwin traced-exec deadlock

Author: XiaoConstantine

pkg/rerank/server_test.go

@@ -56,6 +56,46 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+func TestRunnerDisablesAsyncPreemptionOnlyBeforeTargetExec(t *testing.T) {
+	find := func(environment []string, name string) (string, bool) {
+		prefix := name + "="
+		for _, entry := range environment {
+			if strings.HasPrefix(entry, prefix) {
+				return strings.TrimPrefix(entry, prefix), true
+			}
+		}
+		return "", false
+	}
+	tests := []struct {
+		name         string
+		environment  []string
+		wantDebug    string
+		wantDebugSet bool
+	}{
+		{name: "unset"},
+		{name: "empty", environment: []string{"GODEBUG="}, wantDebugSet: true},
+		{name: "preserved", environment: []string{"GODEBUG=cgocheck=0,asyncpreemptoff=0"}, wantDebug: "cgocheck=0,asyncpreemptoff=0", wantDebugSet: true},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			runner := environmentForRunner(test.environment, "test-instance")
+			runnerDebug, ok := find(runner, "GODEBUG")
+			if !ok || !strings.HasSuffix(runnerDebug, "asyncpreemptoff=1") {
+				t.Fatalf("runner GODEBUG = %q, %v; want asyncpreemptoff=1", runnerDebug, ok)
+			}
+
+			target := environmentForConstrainedReranker(runner)
+			targetDebug, ok := find(target, "GODEBUG")
+			if targetDebug != test.wantDebug || ok != test.wantDebugSet {
+				t.Fatalf("target GODEBUG = %q, %v; want %q, %v", targetDebug, ok, test.wantDebug, test.wantDebugSet)
+			}
+			if _, ok := find(target, supervisorRunnerDebugEnv); ok {
+				t.Fatal("target retained private runner GODEBUG state")
+			}
+		})
+	}
+}
+
 func TestRerankerManagerStopRefusesUnownedLivePID(t *testing.T) {
 	dir := t.TempDir()
 	pid := os.Getpid()
@@ -120,7 +160,8 @@ func TestRerankerManagerStartWorksForEmbeddedHost(t *testing.T) {
 	}
 	t.Cleanup(func() { _ = mgr.Stop() })
 	if err := mgr.Start(); err != nil {
-		t.Fatal(err)
+		log, _ := os.ReadFile(filepath.Join(home, "reranker.log"))
+		t.Fatalf("%v\nreranker log:\n%s", err, log)
 	}
 	state, err := mgr.readPIDState()
 	if err != nil {

pkg/rerank/supervisor.go

@@ -37,6 +37,7 @@ const (
 	supervisorHomeEnv         = "SGREP_RERANKER_SUPERVISOR_HOME"
 	supervisorPortEnv         = "SGREP_RERANKER_SUPERVISOR_PORT"
 	supervisorLifecycleEnv    = "SGREP_RERANKER_SUPERVISOR_LIFECYCLE_LOCK"
+	supervisorRunnerDebugEnv  = "SGREP_RERANKER_SUPERVISOR_RUNNER_GODEBUG"
 	supervisorProtocolVersion = 1
 	guardianCleanShutdown     = byte(0x73)
 	guardianReady             = byte(0x61)
@@ -320,7 +321,7 @@ func runConstrainedReranker(args []string) error {
 	if command[0] != runnerContinue {
 		return fmt.Errorf("invalid constrained reranker authorization")
 	}
-	environment := withoutSupervisorEnvironment(os.Environ())
+	environment := environmentForConstrainedReranker(os.Environ())
 	if err := syscall.Exec(executable, commandArgs, environment); err != nil {
 		return fmt.Errorf("exec constrained reranker: %w", err)
 	}
@@ -1318,12 +1319,52 @@ func environmentForGuardian(environment []string) []string {
 
 func environmentForRunner(environment []string, instance string) []string {
 	result := environmentForReranker(environment, instance)
+	filtered := result[:0]
+	runnerDebug := "asyncpreemptoff=1"
+	for _, entry := range result {
+		if strings.HasPrefix(entry, "GODEBUG=") {
+			original := strings.TrimPrefix(entry, "GODEBUG=")
+			filtered = append(filtered, supervisorRunnerDebugEnv+"="+original)
+			if original != "" {
+				runnerDebug = original + "," + runnerDebug
+			}
+			continue
+		}
+		filtered = append(filtered, entry)
+	}
+	// Darwin's PT_TRACE_ME can delay an asynchronous preemption signal forever,
+	// deadlocking runtime_BeforeExec. The setting is removed before the target
+	// exec, so it applies only to the short-lived Go runner.
+	result = append(filtered, "GODEBUG="+runnerDebug)
 	return append(result,
 		supervisorModeEnv+"=1",
 		supervisorProtocolEnv+"="+strconv.Itoa(supervisorProtocolVersion),
 	)
 }
 
+func environmentForConstrainedReranker(environment []string) []string {
+	var originalDebug string
+	var restoreDebug bool
+	debugPrefix := supervisorRunnerDebugEnv + "="
+	for _, entry := range environment {
+		if strings.HasPrefix(entry, debugPrefix) {
+			originalDebug = strings.TrimPrefix(entry, debugPrefix)
+			restoreDebug = true
+		}
+	}
+	result := withoutSupervisorEnvironment(environment)
+	filtered := result[:0]
+	for _, entry := range result {
+		if !strings.HasPrefix(entry, "GODEBUG=") {
+			filtered = append(filtered, entry)
+		}
+	}
+	if restoreDebug {
+		filtered = append(filtered, "GODEBUG="+originalDebug)
+	}
+	return filtered
+}
+
 func environmentForReranker(environment []string, instance string) []string {
 	result := withoutSupervisorEnvironment(environment)
 	prefix := rerankerInstanceEnv + "="