feat: Speed up bob graph generation hot paths

Batch dependency-sort reachability checks, avoid reflection and channel
overhead in feature and flag hot paths, and skip template parsing for
plain strings.

In the codex-bob-trace-final build directory this reduces build.ninja
generation from the post-struct-cache ~28s range to ~9s in repeated
measurements.

(cherry picked from commit 06e2afbbfaf0027e764ca96762b4dcb005808c8e)
Change-Id: Ib74ce22b4f7a9d155044bdf367ddee17b950d316

Author: jonathan-strandberg-arm

core/dep_sorter.go

@@ -66,14 +66,17 @@ func (handler *graphMutatorHandler) ResolveDependencySortMutator(ctx blueprint.B
 
 	temporaryPaths := map[string][]string{} // For preserving order in declaration
 
-	for i, previous := range mainBuild.Static_libs {
-		for j := i + 1; j < len(mainBuild.Static_libs); j++ {
-			lib := mainBuild.Static_libs[j]
-			if !g.IsReachable(lib, previous) {
-				if g.AddEdge(previous, lib) {
-					temporaryPaths[previous] = append(temporaryPaths[previous], lib)
-					g.SetEdgeColor(previous, lib, "pink")
-				}
+	for j := 1; j < len(mainBuild.Static_libs); j++ {
+		lib := mainBuild.Static_libs[j]
+		reachableFromLib := graph.GetSubgraphNodeSet(g, lib)
+		for i := 0; i < j; i++ {
+			previous := mainBuild.Static_libs[i]
+			if reachableFromLib[previous] {
+				continue
+			}
+			if g.AddEdge(previous, lib) {
+				temporaryPaths[previous] = append(temporaryPaths[previous], lib)
+				g.SetEdgeColor(previous, lib, "pink")
 			}
 		}
 	}

core/feature.go

@@ -229,14 +229,20 @@ func coalesceTypes(list ...reflect.Type) reflect.Type {
 func (f *Features) AppendProps(dst []interface{}, properties *config.Properties) error {
 	// featuresData is struct created in Features.Init function
 	featuresData := reflect.ValueOf(f.BlueprintEmbed).Elem()
+	featuresType := featuresData.Type()
 
-	for _, featureKey := range properties.FeatureList {
+	for i, featureKey := range properties.FeatureList {
 		if properties.Features[featureKey] { // Check the feature is enabled
-			// Features are matched like "Feature_name" - feature structure
-			featureFieldName := featurePropertyName(featureKey)
-			featureStruct := featuresData.FieldByName(featureFieldName)
-			if !featureStruct.IsValid() {
-				utils.Die("Field returned for property %s isn't valid\n", featureFieldName)
+			if i >= featuresData.NumField() {
+				utils.Die("Field returned for property %s isn't valid\n", featureKey)
+			}
+
+			// featureStructType creates fields in FeatureList order. Indexing avoids
+			// a reflective name lookup for every enabled feature on every module.
+			featureStruct := featuresData.Field(i)
+			expectedFieldName := featurePropertyName(featureKey)
+			if featuresType.Field(i).Name != expectedFieldName {
+				utils.Die("Field returned for property %s isn't valid\n", expectedFieldName)
 			}
 
 			// If featureProps is nil then we've determined that we can skip this,

core/flag/flag_collection.go

@@ -5,8 +5,9 @@ package flag
 type Flags []Flag
 
 func (fs Flags) Contains(query Flag) bool {
+	queryString := query.ToString()
 	for _, f := range fs {
-		if f.ToString() == query.ToString() {
+		if f.ToString() == queryString {
 			return true
 		}
 	}
@@ -83,14 +84,16 @@ func (fs Flags) IteratePredicate(predicate func(Flag) bool) <-chan Flag {
 }
 
 func (fs Flags) ForEach(functor func(Flag)) {
-	for f := range fs.Iterate() {
+	for _, f := range fs {
 		functor(f)
 	}
 }
 
 func (fs Flags) ForEachIf(predicate func(Flag) bool, functor func(Flag)) {
-	for f := range fs.IteratePredicate(predicate) {
-		functor(f)
+	for _, f := range fs {
+		if predicate(f) {
+			functor(f)
+		}
 	}
 }
 

core/flag/flag_consumer.go

@@ -22,6 +22,15 @@ func ReferenceFlagsInTransitive(ctx blueprint.BaseModuleContext) (ret Flags) {
 	// * Transitive exports. These make `reexport_libs` obsolete.
 
 	visited := map[string]bool{}
+	addedFlags := map[string]bool{}
+	appendIfUnique := func(f Flag) {
+		flagString := f.ToString()
+		if addedFlags[flagString] {
+			return
+		}
+		addedFlags[flagString] = true
+		ret = append(ret, f)
+	}
 
 	ctx.VisitDirectDeps(func(child blueprint.Module) {
 		if visited[child.Name()] {
@@ -36,9 +45,7 @@ func ReferenceFlagsInTransitive(ctx blueprint.BaseModuleContext) (ret Flags) {
 				},
 			)
 
-			flags.ForEach(func(f Flag) {
-				ret = ret.AppendIfUnique(f)
-			})
+			flags.ForEach(appendIfUnique)
 		}
 	})
 
@@ -55,9 +62,7 @@ func ReferenceFlagsInTransitive(ctx blueprint.BaseModuleContext) (ret Flags) {
 				},
 			)
 
-			flags.ForEach(func(f Flag) {
-				ret = ret.AppendIfUnique(f)
-			})
+			flags.ForEach(appendIfUnique)
 		}
 
 		return true

core/flag/flag_property_parser.go

@@ -2,6 +2,7 @@ package flag
 
 import (
 	"reflect"
+	"sync"
 
 	"github.com/google/blueprint"
 )
@@ -14,13 +15,61 @@ type FlagParserTableEntry struct {
 
 type FlagParserTable []FlagParserTableEntry
 
+var fieldIndexCache = struct {
+	sync.RWMutex
+	indexes map[fieldIndexCacheKey][]int
+}{
+	indexes: make(map[fieldIndexCacheKey][]int),
+}
+
+type fieldIndexCacheKey struct {
+	typ  reflect.Type
+	name string
+}
+
+func concreteValue(v reflect.Value) reflect.Value {
+	for v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	return v
+}
+
+func cachedFieldByName(v reflect.Value, name string) reflect.Value {
+	v = concreteValue(v)
+	key := fieldIndexCacheKey{
+		typ:  v.Type(),
+		name: name,
+	}
+
+	fieldIndexCache.RLock()
+	index, ok := fieldIndexCache.indexes[key]
+	fieldIndexCache.RUnlock()
+	if !ok {
+		field, found := key.typ.FieldByName(name)
+		if !found {
+			panic("missing flag property field: " + name)
+		}
+		index = append([]int(nil), field.Index...)
+
+		fieldIndexCache.Lock()
+		if cached, ok := fieldIndexCache.indexes[key]; ok {
+			index = cached
+		} else {
+			fieldIndexCache.indexes[key] = index
+		}
+		fieldIndexCache.Unlock()
+	}
+
+	return v.FieldByIndex(index)
+}
+
 // Helper method to scrape many properties from a module struct.
 func ParseFromProperties(owner blueprint.Module, luts FlagParserTable, s interface{}) (ret Flags) {
+	v := reflect.ValueOf(s)
 	for _, entry := range luts {
-		s := reflect.Indirect(reflect.ValueOf(&s))
-		v := s.Elem().FieldByName(entry.PropertyName)
-		for _, s := range v.Interface().([]string) {
-			ret = append(ret, entry.Factory(s, owner, entry.Tag))
+		values := cachedFieldByName(v, entry.PropertyName)
+		for i := 0; i < values.Len(); i++ {
+			ret = append(ret, entry.Factory(values.Index(i).String(), owner, entry.Tag))
 		}
 	}
 	return

core/template.go

@@ -53,18 +53,23 @@ func applyTemplateString(elem reflect.Value, stringvalues map[string]string, fun
 		utils.Die("elem is not a string")
 	}
 
+	input := elem.String()
+	if !strings.Contains(input, "{{") {
+		return
+	}
+
 	t := template.New("TemplateProps")
 	t.Option("missingkey=error")
 	t.Funcs(funcmap)
 
-	tmpl, err := t.Parse(elem.String())
+	tmpl, err := t.Parse(input)
 	if err != nil {
-		utils.Die("Error parsing string '%s': %s", elem.String(), err.Error())
+		utils.Die("Error parsing string '%s': %s", input, err.Error())
 	}
 	buf := new(bytes.Buffer)
 	err = tmpl.Execute(buf, stringvalues)
 	if err != nil {
-		utils.Die("Error executing string '%s': %s", elem.String(), err.Error())
+		utils.Die("Error executing string '%s': %s", input, err.Error())
 	}
 	elem.SetString(buf.String())
 }
@@ -86,10 +91,15 @@ func shlexExpand(field reflect.Value, stringvalues map[string]string) {
 	newSlice := make([]string, 0, field.Len())
 	for j := 0; j < field.Len(); j++ {
 		elem := field.Index(j)
+		input := elem.String()
+		if !strings.Contains(input, "{{") {
+			newSlice = append(newSlice, input)
+			continue
+		}
 
-		captures := shlexRegexpr.FindStringSubmatch(elem.String())
+		captures := shlexRegexpr.FindStringSubmatch(input)
 		if len(captures) == 0 {
-			newSlice = append(newSlice, elem.String())
+			newSlice = append(newSlice, input)
 			continue
 		}
 

internal/graph/graph.go

@@ -656,13 +656,19 @@ func walkDownNoCopy(graph Graph, nodeID string, visited map[string]bool,
 
 // A faster alternative to GetSubgraph(graph, start).GetNodeCount()
 func GetSubgraphNodeCount(graph Graph, start string) int {
+	return len(GetSubgraphNodeSet(graph, start))
+}
+
+// A faster alternative to GetSubgraph(graph, start).GetNodes() when only
+// membership queries are needed.
+func GetSubgraphNodeSet(graph Graph, start string) map[string]bool {
 	visited := make(map[string]bool)
 
 	walkDownNoCopy(graph, start, visited,
 		func(Graph, string, map[string]bool) bool { return false },
 		func(Graph, string, string, map[string]bool) bool { return false })
 
-	return len(visited)
+	return visited
 }
 
 // A faster alternative to GetSubgraph(graph, start).HasNode(target)