Skip to content

Commit b77c6af

Browse files
author
Andrzej J Skalski
committed
Also exclude config-level PassUnsafeEnv from remote digest
The previous change only stripped per-target pass_unsafe_env from the cache-key command. Values declared via the global [Build] PassUnsafeEnv config keyword enter the build environment through config.GetBuildEnv() and were still left in the canonical command, so changing them still caused remote cache misses. Strip both config-level and target-level PassUnsafeEnv (keeping anything also listed in PassEnv), and treat a non-empty config-level list as enough to enable the cache-key split. This matches the local cache, which excludes both from its hash.
1 parent df7fa0e commit b77c6af

4 files changed

Lines changed: 101 additions & 14 deletions

File tree

src/remote/action.go

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func (c *Client) buildCommand(target *core.BuildTarget, inputRoot *pb.Directory,
132132
cmd, err := core.ReplaceSequences(state, target, cmd)
133133
env := c.stampedBuildEnvironment(state, target, inputRoot, stamp, isTest || isRun)
134134
if canonical {
135-
c.stripUnsafeEnv(target, env)
135+
c.stripUnsafeEnv(state, target, env)
136136
}
137137
return &pb.Command{
138138
Platform: c.targetPlatformProperties(target),
@@ -142,29 +142,47 @@ func (c *Client) buildCommand(target *core.BuildTarget, inputRoot *pb.Directory,
142142
}, err
143143
}
144144

145-
// excludesUnsafeEnv reports whether PassUnsafeEnv values should be kept out of the action digest for this target.
146-
func (c *Client) excludesUnsafeEnv(target *core.BuildTarget) bool {
147-
return c.state.Config.Remote.ExcludePassUnsafeEnvVarsFromDigest && target.PassUnsafeEnv != nil && len(*target.PassUnsafeEnv) > 0
145+
// excludesUnsafeEnv reports whether PassUnsafeEnv values should be kept out of the action digest for this
146+
// target. This considers both the global [Build] PassUnsafeEnv config keyword and the target's own
147+
// pass_unsafe_env attribute, mirroring how the local cache excludes both from its hash.
148+
func (c *Client) excludesUnsafeEnv(state *core.BuildState, target *core.BuildTarget) bool {
149+
if !c.state.Config.Remote.ExcludePassUnsafeEnvVarsFromDigest {
150+
return false
151+
}
152+
if len(state.Config.Build.PassUnsafeEnv) > 0 {
153+
return true
154+
}
155+
return target.PassUnsafeEnv != nil && len(*target.PassUnsafeEnv) > 0
148156
}
149157

150158
// stripUnsafeEnv removes the values of PassUnsafeEnv variables from the given environment so that they do
151-
// not contribute to the action digest. Variables that are also listed in PassEnv are left intact, since
152-
// those values are intentionally part of the cache key.
153-
func (c *Client) stripUnsafeEnv(target *core.BuildTarget, env core.BuildEnv) {
154-
if !c.excludesUnsafeEnv(target) {
159+
// not contribute to the action digest. Both the global [Build] PassUnsafeEnv config keyword and the
160+
// target's pass_unsafe_env attribute are considered. Variables that are also listed in PassEnv (config or
161+
// target level) are left intact, since those values are intentionally part of the cache key.
162+
func (c *Client) stripUnsafeEnv(state *core.BuildState, target *core.BuildTarget, env core.BuildEnv) {
163+
if !c.excludesUnsafeEnv(state, target) {
155164
return
156165
}
157166
safe := map[string]bool{}
167+
for _, e := range state.Config.Build.PassEnv {
168+
safe[e] = true
169+
}
158170
if target.PassEnv != nil {
159171
for _, e := range *target.PassEnv {
160172
safe[e] = true
161173
}
162174
}
163-
for _, e := range *target.PassUnsafeEnv {
164-
if !safe[e] {
165-
delete(env, e)
175+
strip := func(vars []string) {
176+
for _, e := range vars {
177+
if !safe[e] {
178+
delete(env, e)
179+
}
166180
}
167181
}
182+
strip(state.Config.Build.PassUnsafeEnv)
183+
if target.PassUnsafeEnv != nil {
184+
strip(*target.PassUnsafeEnv)
185+
}
168186
}
169187

170188
// stampedBuildEnvironment returns a build environment, optionally with a stamp if stamp is true.
@@ -194,7 +212,7 @@ func (c *Client) buildTestCommand(state *core.BuildState, target *core.BuildTarg
194212
cmd, err := core.ReplaceTestSequences(state, target, target.GetTestCommand(state))
195213
env := core.TestEnvironment(state, target, ".", run)
196214
if canonical {
197-
c.stripUnsafeEnv(target, env)
215+
c.stripUnsafeEnv(state, target, env)
198216
}
199217
return &pb.Command{
200218
Platform: &pb.Platform{

src/remote/impl_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,23 @@ func newClient() *Client {
3434
}
3535

3636
func newClientInstance(name string) *Client {
37+
return newClientInstanceWith(name, nil)
38+
}
39+
40+
// newClientInstanceWith builds a client, optionally applying configure to the configuration before the
41+
// build state (and the client's async initialisation) is created. This matters for config values that are
42+
// captured once, e.g. the build environment derived from PassUnsafeEnv.
43+
func newClientInstanceWith(name string, configure func(*core.Configuration)) *Client {
3744
config := core.DefaultConfiguration()
3845
config.Build.Path = []string{"/usr/local/bin", "/usr/bin", "/bin"}
3946
config.Build.HashFunction = "sha256"
4047
config.Remote.NumExecutors = 1
4148
config.Remote.Instance = name
4249
config.Remote.Secure = false
4350
config.Remote.Platform = []string{"OSFamily=linux"}
51+
if configure != nil {
52+
configure(config)
53+
}
4454
state := core.NewBuildState(config)
4555
state.Config.Remote.URL = "127.0.0.1:9987"
4656
state.Config.Remote.AssetURL = state.Config.Remote.URL

src/remote/remote.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ func (c *Client) build(target *core.BuildTarget) (*core.BuildMetadata, *pb.Actio
417417
// store & look up results under a "cache-key" action digest that omits those inputs, while still
418418
// executing the real action that includes them. This implements the rule whereby e.g. the SCM
419419
// revision or an unsafe env var changing doesn't force a rebuild.
420-
useCacheKeyDigest := target.Stamp || c.excludesUnsafeEnv(target)
420+
useCacheKeyDigest := target.Stamp || c.excludesUnsafeEnv(c.state.ForTarget(target), target)
421421
var cacheKeyDigest *pb.Digest
422422
if useCacheKeyDigest {
423423
command, digest, err := c.buildAction(target, false, false, true, 0)
@@ -614,7 +614,7 @@ func (c *Client) Test(target *core.BuildTarget, run int) (metadata *core.BuildMe
614614
}
615615
// As in build(), if PassUnsafeEnv values are excluded from the digest we look results up under a
616616
// cache-key digest that omits them, while executing the real action that includes them.
617-
useCacheKeyDigest := c.excludesUnsafeEnv(target)
617+
useCacheKeyDigest := c.excludesUnsafeEnv(c.state.ForTarget(target), target)
618618
var ar *pb.ActionResult
619619
if useCacheKeyDigest {
620620
cacheKeyCommand, cacheKeyDigest, cErr := c.buildAction(target, true, false, true, run)

src/remote/remote_test.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,65 @@ func TestPassUnsafeEnvExcludedFromDigest(t *testing.T) {
267267
assert.True(t, envContainsValue(real1, "first"))
268268
}
269269

270+
// TestStripUnsafeEnvConfigLevel checks the stripping logic removes values from the global [Build]
271+
// PassUnsafeEnv config keyword while retaining PassEnv values (which are intentionally part of the cache key).
272+
func TestStripUnsafeEnvConfigLevel(t *testing.T) {
273+
c := newClientInstance("strip")
274+
c.state.Config.Remote.ExcludePassUnsafeEnvVarsFromDigest = true
275+
c.state.Config.Build.PassUnsafeEnv = []string{"CFG_UNSAFE"}
276+
c.state.Config.Build.PassEnv = []string{"CFG_SAFE"}
277+
278+
target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "x"})
279+
unsafe := []string{"TGT_UNSAFE"}
280+
target.PassUnsafeEnv = &unsafe
281+
282+
env := core.BuildEnv{
283+
"CFG_UNSAFE": "secret",
284+
"CFG_SAFE": "keep",
285+
"TGT_UNSAFE": "alsosecret",
286+
"OTHER": "v",
287+
}
288+
c.stripUnsafeEnv(c.state.ForTarget(target), target, env)
289+
290+
_, cfgUnsafePresent := env["CFG_UNSAFE"]
291+
_, tgtUnsafePresent := env["TGT_UNSAFE"]
292+
assert.False(t, cfgUnsafePresent, "config-level PassUnsafeEnv should be stripped")
293+
assert.False(t, tgtUnsafePresent, "target-level PassUnsafeEnv should be stripped")
294+
assert.Equal(t, "keep", env["CFG_SAFE"], "PassEnv must be retained")
295+
assert.Equal(t, "v", env["OTHER"], "unrelated env must be retained")
296+
}
297+
298+
// TestConfigPassUnsafeEnvExcludedFromDigest checks that values declared via the global [Build]
299+
// PassUnsafeEnv config keyword (not just the per-target attribute) are excluded from the cache-key digest.
300+
// Because config-level values are captured once per config object, this uses a separate client per value
301+
// and sets the config before the client (and its async init) is created.
302+
func TestConfigPassUnsafeEnvExcludedFromDigest(t *testing.T) {
303+
canonicalAndReal := func(value string) (string, string) {
304+
t.Setenv("MY_CFG_UNSAFE", value)
305+
c := newClientInstanceWith(fmt.Sprintf("cfg-unsafe-%d", time.Now().UnixNano()), func(config *core.Configuration) {
306+
config.Remote.ExcludePassUnsafeEnvVarsFromDigest = true
307+
config.Build.PassUnsafeEnv = []string{"MY_CFG_UNSAFE"}
308+
})
309+
target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "cfgunsafe"})
310+
target.AddOutput("out.txt")
311+
target.Command = "echo hello > $OUT"
312+
target.BuildTimeout = time.Minute
313+
canon, canonDigest, err := c.buildAction(target, false, false, true, 0)
314+
require.NoError(t, err)
315+
require.False(t, envContainsValue(canon, value), "config-level PassUnsafeEnv value must not be in the canonical command")
316+
real, realDigest, err := c.buildAction(target, false, true, false, 0)
317+
require.NoError(t, err)
318+
require.True(t, envContainsValue(real, value), "executed action should include the config-level value")
319+
return canonDigest.Hash, realDigest.Hash
320+
}
321+
322+
canonFirst, realFirst := canonicalAndReal("first")
323+
canonSecond, realSecond := canonicalAndReal("second")
324+
325+
assert.Equal(t, canonFirst, canonSecond, "config-level PassUnsafeEnv value must not affect the cache-key digest")
326+
assert.NotEqual(t, realFirst, realSecond, "the executed action should still include the config-level value")
327+
}
328+
270329
// TestPassUnsafeEnvRemoteCacheHitAcrossValues is an end-to-end test against the in-process testServer
271330
// that proves changing a PassUnsafeEnv value does not re-execute the action: the first build executes
272331
// and backfills the cache-key digest, and a second build with a *different* value (and a fresh client,

0 commit comments

Comments
 (0)