Search float stack dimensions for self-recursive word effects

The self-recursive fixpoint search only tried data stack dimensions.
Words like trace-ray that take float inputs ({1, 0, 6, 3}) were
rejected. Now the search also covers FIn and FOut up to the float
depth at the first RECURSE call.

Author: ncw

pkg/compiler/stackeffect.go

@@ -314,7 +314,7 @@ func analyzeWithSelfRef(
 	ds := depthState{depth: 0, minDepth: 0}
 	pendingMerges := map[int]depthState{}
 	dead := false
-	maxIn := 0
+	maxIn, maxFIn := 0, 0
 	for i := 0; i < len(entry.Body); i++ {
 		if saved, ok := pendingMerges[i]; ok {
 			delete(pendingMerges, i)
@@ -328,34 +328,41 @@ func analyzeWithSelfRef(
 		}
 		item := entry.Body[i]
 		if item.Word != nil && item.Word == entry {
-			// depth at RECURSE + items consumed below entry = max possible In
 			in := 0
 			if ds.minDepth < 0 {
 				in = -ds.minDepth
 			}
 			maxIn = ds.depth + in
+			fin := 0
+			if ds.fminDepth < 0 {
+				fin = -ds.fminDepth
+			}
+			maxFIn = ds.fdepth + fin
 			break
 		}
 		_ = a.processItem(item, i, &ds, pendingMerges, &dead)
 	}
-	if maxIn == 0 {
-		maxIn = 4 // fallback
+	if maxIn == 0 && maxFIn == 0 {
+		maxIn = 4
 	}
 
-	// Try all combinations {in, out} up to maxIn.
-	// Most recursive words have In == Out or Out < In.
+	// Try all combinations up to the bounds found at the RECURSE call.
 	for in := 0; in <= maxIn; in++ {
 		for out := 0; out <= maxIn; out++ {
-			candidate := StackEffect{in, out, 0, 0}
-			tempKnown[entry.Xt] = candidate
-			av := &effectAnalyzer{
-				body:         entry.Body,
-				knownEffects: tempKnown,
-				wordName:     entry.Name,
-			}
-			result, err := av.analyze()
-			if err == nil && result == candidate {
-				return result, nil
+			for fin := 0; fin <= maxFIn; fin++ {
+				for fout := 0; fout <= maxFIn; fout++ {
+					candidate := StackEffect{in, out, fin, fout}
+					tempKnown[entry.Xt] = candidate
+					av := &effectAnalyzer{
+						body:         entry.Body,
+						knownEffects: tempKnown,
+						wordName:     entry.Name,
+					}
+					result, err := av.analyze()
+					if err == nil && result == candidate {
+						return result, nil
+					}
+				}
 			}
 		}
 	}