feat(pkg): scope queries to incremental dirty targets

Author: mattnworb

pkg/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "hash_cache.go",
         "hash_persistence.go",
         "normalizer.go",
+        "scoped_query.go",
         "target_determinator.go",
         "targets_list.go",
         "walker.go",
@@ -41,6 +42,7 @@ go_test(
         "hash_cache_test.go",
         "hash_persistence_test.go",
         "normalizer_test.go",
+        "scoped_query_test.go",
         "seed_hashes_test.go",
         "target_determinator_test.go",
         "walker_test.go",

pkg/scoped_query.go

@@ -0,0 +1,89 @@
+package pkg
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// BuildScopedUniverse returns a bazel query expression covering the scoped
+// universe of a seeded run:
+//
+//   - each dirty package contributes a `//pkg:all` wildcard, so that targets
+//     added to or removed from the package since the seed are picked up
+//     (explicit labels from the seed would miss new targets and error on
+//     deleted ones);
+//   - carried labels — dirty* targets in unchanged packages (i.e. reverse
+//     deps of the dirty packages) — are named explicitly via set(). These
+//     labels are guaranteed to still exist because their BUILD files did not
+//     change and macro (.bzl) changes trigger a full-rehash fallback.
+//
+// The returned expression is parenthesized so it can be substituted into a
+// larger query expression.
+func BuildScopedUniverse(dirtyPackages []string, carriedLabels []string) string {
+	terms := make([]string, 0, len(dirtyPackages)+1)
+	for _, pkg := range dirtyPackages {
+		if pkg == "//" {
+			terms = append(terms, "//:all")
+		} else {
+			terms = append(terms, pkg+":all")
+		}
+	}
+	if len(carriedLabels) > 0 {
+		sorted := append([]string(nil), carriedLabels...)
+		sort.Strings(sorted)
+		terms = append(terms, "set("+strings.Join(sorted, " ")+")")
+	}
+	if len(terms) == 0 {
+		return "set()"
+	}
+	return "(" + strings.Join(terms, " + ") + ")"
+}
+
+// ScopeTargetsPattern narrows a targets pattern built over //... to the given
+// universe expression by substituting every occurrence of //... with the
+// universe. This preserves the semantics of filters expressed over the full
+// repo — e.g. `//... - attr(tags, "manual", //...)` becomes
+// `(U) - attr(tags, "manual", (U))` — while restricting evaluation to the
+// scoped universe, so manual-tag filtering behaves identically to a full run.
+//
+// Patterns that do not contain //... cannot be scoped this way; callers
+// should fall back to a full computation in that case.
+//
+// Known limitations of scoped mode versus a full //... query:
+//   - --filter-incompatible-targets is a cquery-only concern; the seeded path
+//     is only supported with --query-backend=query, which never filters
+//     incompatible targets in full mode either.
+//   - Targets in unchanged packages whose rule-generation depends on state
+//     outside the package (e.g. macros) are not re-listed; any .bzl change
+//     triggers a full-rehash fallback before reaching this point.
+func ScopeTargetsPattern(originalPattern string, universe string) (string, error) {
+	var result strings.Builder
+	replacements := 0
+	for start := 0; start < len(originalPattern); {
+		idx := strings.Index(originalPattern[start:], "//...")
+		if idx < 0 {
+			result.WriteString(originalPattern[start:])
+			break
+		}
+		idx += start
+		result.WriteString(originalPattern[start:idx])
+		// Do not rewrite repository-qualified wildcards such as @repo//....
+		if idx > 0 && isLabelCharacter(originalPattern[idx-1]) {
+			result.WriteString("//...")
+		} else {
+			result.WriteString(universe)
+			replacements++
+		}
+		start = idx + len("//...")
+	}
+	if replacements == 0 {
+		return "", fmt.Errorf("targets pattern %q does not contain //... and cannot be scoped", originalPattern)
+	}
+	return result.String(), nil
+}
+
+func isLabelCharacter(c byte) bool {
+	return c == '@' || c == '_' || c == '-' || c == '.' || c == '+' ||
+		(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')
+}

pkg/scoped_query_test.go

@@ -0,0 +1,82 @@
+package pkg
+
+import (
+	"testing"
+)
+
+func TestBuildScopedUniverseEmpty(t *testing.T) {
+	got := BuildScopedUniverse(nil, nil)
+	if got != "set()" {
+		t.Errorf("empty universe: want %q, got %q", "set()", got)
+	}
+}
+
+func TestBuildScopedUniversePackagesOnly(t *testing.T) {
+	got := BuildScopedUniverse([]string{"//a", "//b/c"}, nil)
+	want := "(//a:all + //b/c:all)"
+	if got != want {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}
+
+func TestBuildScopedUniverseRootPackage(t *testing.T) {
+	got := BuildScopedUniverse([]string{"//"}, nil)
+	if got != "(//:all)" {
+		t.Fatalf("BuildScopedUniverse(root) = %q, want %q", got, "(//:all)")
+	}
+}
+
+func TestBuildScopedUniverseLabelsOnly(t *testing.T) {
+	got := BuildScopedUniverse(nil, []string{"//z:z", "//a:a"})
+	want := "(set(//a:a //z:z))"
+	if got != want {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}
+
+func TestBuildScopedUniversePackagesAndLabels(t *testing.T) {
+	got := BuildScopedUniverse([]string{"//pkg"}, []string{"//app:binary"})
+	want := "(//pkg:all + set(//app:binary))"
+	if got != want {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}
+
+func TestScopeTargetsPatternPlain(t *testing.T) {
+	got, err := ScopeTargetsPattern("//...", "(//pkg:all)")
+	if err != nil {
+		t.Fatalf("ScopeTargetsPattern: %v", err)
+	}
+	if got != "(//pkg:all)" {
+		t.Errorf("want %q, got %q", "(//pkg:all)", got)
+	}
+}
+
+func TestScopeTargetsPatternManualFilter(t *testing.T) {
+	original := `//... - attr(tags, "manual", //...)`
+	got, err := ScopeTargetsPattern(original, "(//pkg:all + set(//app:binary))")
+	if err != nil {
+		t.Fatalf("ScopeTargetsPattern: %v", err)
+	}
+	want := `(//pkg:all + set(//app:binary)) - attr(tags, "manual", (//pkg:all + set(//app:binary)))`
+	if got != want {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}
+
+func TestScopeTargetsPatternNoWildcard(t *testing.T) {
+	if _, err := ScopeTargetsPattern("//app:binary", "(//pkg:all)"); err == nil {
+		t.Fatal("expected error for pattern without //...")
+	}
+}
+
+func TestScopeTargetsPatternDoesNotRewriteExternalWildcard(t *testing.T) {
+	got, err := ScopeTargetsPattern("@repo//... + //...", "(//pkg:all)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "@repo//... + (//pkg:all)"
+	if got != want {
+		t.Fatalf("ScopeTargetsPattern = %q, want %q", got, want)
+	}
+}

pkg/target_determinator.go

@@ -1115,8 +1115,46 @@ func runToCqueryResult(context *Context, pattern string, includeTransitions bool
 	}
 }
 
+// maxInlineQueryPatternLength is the longest query pattern passed to bazel as
+// a command-line argument. Longer patterns (e.g. scoped seeded-mode universes
+// carrying many explicit labels) are written to a file and passed via
+// --query_file to avoid OS argv limits.
+const maxInlineQueryPatternLength = 65536
+
+// queryPatternArgs returns the bazel query arguments encoding pattern, either
+// inline or via a --query_file, along with a cleanup function.
+func queryPatternArgs(pattern string) ([]string, func(), error) {
+	if len(pattern) <= maxInlineQueryPatternLength {
+		return []string{pattern}, func() {}, nil
+	}
+	queryFile, err := os.CreateTemp("", "target-determinator-query-pattern-*.txt")
+	if err != nil {
+		return nil, func() {}, fmt.Errorf("failed to create query pattern file: %w", err)
+	}
+	cleanup := func() { os.Remove(queryFile.Name()) }
+	if _, err := queryFile.WriteString(pattern); err != nil {
+		queryFile.Close()
+		cleanup()
+		return nil, func() {}, fmt.Errorf("failed to write query pattern file: %w", err)
+	}
+	if err := queryFile.Close(); err != nil {
+		cleanup()
+		return nil, func() {}, fmt.Errorf("failed to close query pattern file: %w", err)
+	}
+	return []string{"--query_file=" + queryFile.Name()}, cleanup, nil
+}
+
+// truncateForLog shortens very long query patterns for log lines.
+func truncateForLog(pattern string) string {
+	const maxLogged = 2048
+	if len(pattern) <= maxLogged {
+		return pattern
+	}
+	return pattern[:maxLogged] + fmt.Sprintf("... (%d bytes total)", len(pattern))
+}
+
 func runToQueryResult(context *Context, pattern string) ([]*analysis.ConfiguredTarget, error) {
-	log.Printf("Running query on %s", pattern)
+	log.Printf("Running query on %s", truncateForLog(pattern))
 
 	queryOutputFile, err := os.CreateTemp("", "target-determinator-query-*.proto")
 	if err != nil {
@@ -1128,17 +1166,23 @@ func runToQueryResult(context *Context, pattern string) ([]*analysis.ConfiguredT
 		return nil, fmt.Errorf("failed to close temporary file for query output: %w", err)
 	}
 
+	patternArgs, patternCleanup, err := queryPatternArgs(pattern)
+	defer patternCleanup()
+	if err != nil {
+		return nil, err
+	}
+
 	var stderr bytes.Buffer
 	// Unlike cquery (which only gained streamed_proto and output_file in Bazel 8.2),
 	// bazel query has supported both since well before any Bazel version this tool targets.
 	returnVal, err := context.BazelCmd.Execute(
 		BazelCmdConfig{Dir: context.WorkspacePath, Stderr: &stderr},
 		[]string{"--output_base", context.BazelOutputBase},
-		"query", "--output=streamed_proto", "--order_output=no",
-		"--output_file="+queryOutput, pattern)
+		"query", append([]string{"--output=streamed_proto", "--order_output=no",
+			"--output_file=" + queryOutput}, patternArgs...)...)
 
 	if returnVal != 0 || err != nil {
-		return nil, fmt.Errorf("failed to run query on %s: %w. Stderr:\n%v", pattern, err, stderr.String())
+		return nil, fmt.Errorf("failed to run query on %s: %w. Stderr:\n%v", truncateForLog(pattern), err, stderr.String())
 	}
 
 	queryOutputFile, err = os.Open(queryOutput)
@@ -1167,17 +1211,23 @@ func runToQueryResult(context *Context, pattern string) ([]*analysis.ConfiguredT
 }
 
 func runToQueryLabels(context *Context, pattern string, normalizer *Normalizer) ([]label.Label, error) {
-	log.Printf("Running query (labels) on %s", pattern)
+	log.Printf("Running query (labels) on %s", truncateForLog(pattern))
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 
+	patternArgs, patternCleanup, err := queryPatternArgs(pattern)
+	defer patternCleanup()
+	if err != nil {
+		return nil, err
+	}
+
 	returnVal, err := context.BazelCmd.Execute(
 		BazelCmdConfig{Dir: context.WorkspacePath, Stdout: &stdout, Stderr: &stderr},
 		[]string{"--output_base", context.BazelOutputBase},
-		"query", "--output=label", "--order_output=no", pattern)
+		"query", append([]string{"--output=label", "--order_output=no"}, patternArgs...)...)
 
 	if returnVal != 0 || err != nil {
-		return nil, fmt.Errorf("failed to run query on %s: %w. Stderr:\n%v", pattern, err, stderr.String())
+		return nil, fmt.Errorf("failed to run query on %s: %w. Stderr:\n%v", truncateForLog(pattern), err, stderr.String())
 	}
 
 	var labels []label.Label