Skip to content

Commit bdbe355

Browse files
authored
Merge pull request #4574 from The-OpenROAD-Project-staging/upload-build-requester
Send who requested the build in the metrics payload
2 parents 61280c2 + 5a13da8 commit bdbe355

2 files changed

Lines changed: 253 additions & 5 deletions

File tree

‎flow/test/test_uploadMetadata.py‎

Lines changed: 182 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ def from_service_account_file(path):
6161

6262
BUILD_TIME_FORMAT = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$")
6363

64+
REQUESTER_ENV = ("UPLOAD_BUILD_USER_EMAIL", "UPLOAD_CHANGE_AUTHOR_EMAIL")
65+
66+
# Large enough to push the pipeline payload over the 8 MB cap, so the script
67+
# falls back to one v1 message per design.
68+
OVERSIZED_METRICS = {"blob": "x" * (9 * 1024 * 1024)}
69+
6470

6571
class TestUploadMetadata(unittest.TestCase):
6672
def setUp(self):
@@ -96,8 +102,11 @@ def add_design(self, platform, design, variant="base", metrics=None):
96102
with open(os.path.join(report_dir, "metadata.json"), "w") as f:
97103
json.dump(metrics or {"finish:area": 1.0}, f)
98104

99-
def run_script(self, *extra, fail=False, tz=None):
105+
def run_script(self, *extra, fail=False, tz=None, extra_env=None):
100106
env = dict(os.environ)
107+
for name in REQUESTER_ENV:
108+
env.pop(name, None)
109+
env.update(extra_env or {})
101110
if tz:
102111
env["TZ"] = tz
103112
env["PYTHONPATH"] = self.stubs
@@ -343,6 +352,178 @@ def test_empty_designs_valid_provenance_is_v4(self):
343352
self.assertEqual(payload["designs"], [])
344353
self.assertEqual(payload["jenkins_library"], {"ref": "v1.0.0"})
345354

355+
def test_pipeline_payload_without_requester_is_unchanged(self):
356+
self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0})
357+
result, messages = self.run_script("--jobName", "ORFS")
358+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
359+
payload = messages[0]["data"]
360+
self.assertRegex(payload.pop("build_time"), BUILD_TIME_FORMAT)
361+
self.assertEqual(
362+
payload,
363+
{
364+
"payload_schema_version": 3,
365+
"jenkins_env": "public",
366+
"build_id": "42",
367+
"branch_name": "master",
368+
"pipeline_id": "jenkins-ORFS-master-42",
369+
"change_branch": None,
370+
"commit_sha": "abc123",
371+
"jenkins_url": "http://jenkins/job/ORFS/42/",
372+
"job_name": "ORFS",
373+
"designs": [
374+
{
375+
"platform": "nangate45",
376+
"design": "gcd",
377+
"variant": "base",
378+
"metrics": {"finish__area": 5.0},
379+
}
380+
],
381+
},
382+
)
383+
384+
def test_v1_message_without_requester_is_unchanged(self):
385+
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
386+
result, messages = self.run_script()
387+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
388+
self.assertEqual(len(messages), 1)
389+
payload = messages[0]["data"]
390+
self.assertEqual(payload.pop("blob"), OVERSIZED_METRICS["blob"])
391+
self.assertEqual(
392+
payload,
393+
{
394+
"build_id": "42",
395+
"branch_name": "master",
396+
"pipeline_id": "jenkins-ORFS-master-42",
397+
"change_branch": None,
398+
"commit_sha": "abc123",
399+
"jenkins_url": "http://jenkins/job/ORFS/42/",
400+
"jenkins_env": "public",
401+
},
402+
)
403+
404+
def test_both_requester_flags_reach_pipeline_payload(self):
405+
self.add_design("nangate45", "gcd")
406+
result, messages = self.run_script(
407+
"--buildUserEmail",
408+
" ada@example.com ",
409+
"--changeAuthorEmail",
410+
"bob@example.com",
411+
)
412+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
413+
payload = messages[0]["data"]
414+
self.assertEqual(payload["build_user_email"], "ada@example.com")
415+
self.assertEqual(payload["change_author_email"], "bob@example.com")
416+
self.assertEqual(payload["payload_schema_version"], 2)
417+
418+
def test_both_requester_flags_reach_v1_messages(self):
419+
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
420+
result, messages = self.run_script(
421+
"--buildUserEmail",
422+
"ada@example.com",
423+
"--changeAuthorEmail",
424+
"bob@example.com",
425+
)
426+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
427+
payload = messages[0]["data"]
428+
self.assertEqual(payload["build_user_email"], "ada@example.com")
429+
self.assertEqual(payload["change_author_email"], "bob@example.com")
430+
431+
def test_blank_requester_flag_is_omitted(self):
432+
self.add_design("nangate45", "gcd")
433+
result, messages = self.run_script(
434+
"--buildUserEmail", " ", "--changeAuthorEmail", "bob@example.com"
435+
)
436+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
437+
payload = messages[0]["data"]
438+
self.assertNotIn("build_user_email", payload)
439+
self.assertEqual(payload["change_author_email"], "bob@example.com")
440+
441+
def test_blank_requester_flag_is_omitted_from_v1_messages(self):
442+
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
443+
result, messages = self.run_script(
444+
"--buildUserEmail", "ada@example.com", "--changeAuthorEmail", ""
445+
)
446+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
447+
payload = messages[0]["data"]
448+
self.assertEqual(payload["build_user_email"], "ada@example.com")
449+
self.assertNotIn("change_author_email", payload)
450+
451+
def test_requester_without_at_sign_is_dropped(self):
452+
self.add_design("nangate45", "gcd")
453+
result, messages = self.run_script(
454+
"--buildUserEmail", "null", "--changeAuthorEmail", "None"
455+
)
456+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
457+
payload = messages[0]["data"]
458+
self.assertNotIn("build_user_email", payload)
459+
self.assertNotIn("change_author_email", payload)
460+
self.assertIn("[WARN] Not sending --buildUserEmail: 'null'", result.stdout)
461+
self.assertIn("[WARN] Not sending --changeAuthorEmail: 'None'", result.stdout)
462+
463+
def test_v1_fallback_warns_about_requester_once(self):
464+
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
465+
result, messages = self.run_script("--buildUserEmail", "null")
466+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
467+
self.assertNotIn("build_user_email", messages[0]["data"])
468+
self.assertEqual(result.stdout.count("Not sending --buildUserEmail"), 1)
469+
470+
def test_requester_is_read_from_environment(self):
471+
self.add_design("nangate45", "gcd")
472+
result, messages = self.run_script(
473+
extra_env={
474+
"UPLOAD_BUILD_USER_EMAIL": "ada@example.com",
475+
"UPLOAD_CHANGE_AUTHOR_EMAIL": " bob@example.com ",
476+
}
477+
)
478+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
479+
payload = messages[0]["data"]
480+
self.assertEqual(payload["build_user_email"], "ada@example.com")
481+
self.assertEqual(payload["change_author_email"], "bob@example.com")
482+
483+
def test_requester_flag_overrides_environment(self):
484+
self.add_design("nangate45", "gcd")
485+
result, messages = self.run_script(
486+
"--buildUserEmail",
487+
"ada@example.com",
488+
extra_env={"UPLOAD_BUILD_USER_EMAIL": "other@example.com"},
489+
)
490+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
491+
self.assertEqual(messages[0]["data"]["build_user_email"], "ada@example.com")
492+
493+
def test_empty_requester_flag_suppresses_environment(self):
494+
self.add_design("nangate45", "gcd")
495+
result, messages = self.run_script(
496+
"--buildUserEmail",
497+
"",
498+
extra_env={"UPLOAD_BUILD_USER_EMAIL": "ada@example.com"},
499+
)
500+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
501+
self.assertNotIn("build_user_email", messages[0]["data"])
502+
503+
def test_environment_requester_is_validated(self):
504+
self.add_design("nangate45", "gcd")
505+
result, messages = self.run_script(
506+
extra_env={"UPLOAD_BUILD_USER_EMAIL": "null"}
507+
)
508+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
509+
self.assertNotIn("build_user_email", messages[0]["data"])
510+
self.assertIn("[WARN] Not sending --buildUserEmail: 'null'", result.stdout)
511+
512+
def test_environment_requester_reaches_v1_messages(self):
513+
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
514+
result, messages = self.run_script(
515+
extra_env={"UPLOAD_CHANGE_AUTHOR_EMAIL": "bob@example.com"}
516+
)
517+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
518+
self.assertEqual(messages[0]["data"]["change_author_email"], "bob@example.com")
519+
520+
def test_requester_keys_win_over_same_named_v1_metric(self):
521+
metrics = dict(OVERSIZED_METRICS, build_user_email="metric")
522+
self.add_design("nangate45", "gcd", metrics=metrics)
523+
result, messages = self.run_script("--buildUserEmail", "ada@example.com")
524+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
525+
self.assertEqual(messages[0]["data"]["build_user_email"], "ada@example.com")
526+
346527
def test_no_publisher_publishes_nothing(self):
347528
env = dict(os.environ)
348529
env["PYTHONPATH"] = self.stubs

‎flow/util/uploadMetadata.py‎

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,24 @@ def _non_negative_int(val):
111111
"a warning). Emitted as no_metrics_reason only on a payload with no "
112112
"designs.",
113113
)
114+
parser.add_argument(
115+
"--buildUserEmail",
116+
type=str,
117+
default=None,
118+
help="Email of the Jenkins user who started the build by hand "
119+
"(BUILD_USER_EMAIL). Omitted for SCM, timer and branch-scan triggers. "
120+
"The dashboard attributes the build to this user. Defaults to "
121+
"$UPLOAD_BUILD_USER_EMAIL, which keeps the address out of a traced "
122+
"(sh -x) command line.",
123+
)
124+
parser.add_argument(
125+
"--changeAuthorEmail",
126+
type=str,
127+
default=None,
128+
help="Email of the pull request's author on a PR build "
129+
"(CHANGE_AUTHOR_EMAIL). Empty when the author's GitHub email is private. "
130+
"Defaults to $UPLOAD_CHANGE_AUTHOR_EMAIL.",
131+
)
114132

115133
# --- PUBSUB args ---
116134
parser.add_argument(
@@ -253,6 +271,45 @@ def resolve_schema_version(args, provenance):
253271
return 4 if provenance else 3
254272

255273

274+
def resolve_requester(args):
275+
"""Who asked for the build, as the payload keys the backend reads.
276+
277+
Each value comes from its flag or, when the flag is absent, from its
278+
environment variable. An empty flag counts as given, so it suppresses the
279+
variable. Blank values are left out rather than sent empty, and a value
280+
without an "@" is dropped with a warning: an unset Groovy variable
281+
interpolates as the string "null", which must not be stored as an email.
282+
The keys ride on every schema version: the backend reads them by presence,
283+
and they describe the build rather than the payload's shape, so they do
284+
not bump the version.
285+
"""
286+
requester = {}
287+
for key, flag, value, env_var in (
288+
(
289+
"build_user_email",
290+
"--buildUserEmail",
291+
args.buildUserEmail,
292+
"UPLOAD_BUILD_USER_EMAIL",
293+
),
294+
(
295+
"change_author_email",
296+
"--changeAuthorEmail",
297+
args.changeAuthorEmail,
298+
"UPLOAD_CHANGE_AUTHOR_EMAIL",
299+
),
300+
):
301+
if value is None:
302+
value = os.environ.get(env_var, "")
303+
value = value.strip()
304+
if not value:
305+
continue
306+
if "@" not in value:
307+
print(f"[WARN] Not sending {flag}: '{value}' is not an email address.")
308+
continue
309+
requester[key] = value
310+
return requester
311+
312+
256313
def resolve_build_time(args):
257314
"""The build time to report, defaulting to now in genMetrics.py's format.
258315
@@ -283,7 +340,7 @@ def resolve_no_metrics_reason(args, design_records):
283340

284341

285342
def build_pipeline_payload(
286-
design_records, args, provenance=None, no_metrics_reason=None
343+
design_records, args, provenance=None, no_metrics_reason=None, requester=None
287344
):
288345
"""Return the pipeline-level payload dict.
289346
@@ -316,6 +373,8 @@ def build_pipeline_payload(
316373
# the payload.
317374
if schema_version == 4:
318375
payload.update(provenance)
376+
if requester:
377+
payload.update(requester)
319378
return payload
320379

321380

@@ -339,7 +398,9 @@ def publish_pipeline_report(
339398
print(f"[INFO] Published pipeline report to Pub/Sub (message ID: {message_id}).")
340399

341400

342-
def publish_v1_per_design(publisher, topic_path, design_records, args, provenance=None):
401+
def publish_v1_per_design(
402+
publisher, topic_path, design_records, args, provenance=None, requester=None
403+
):
343404
"""Fallback path used when the v2 pipeline payload exceeds MAX_PUBSUB_BYTES.
344405
345406
Emits one v1-format message per design (no payload_schema_version, metrics
@@ -362,6 +423,9 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc
362423
metadata there but not needed: each design's metrics carry
363424
run__flow__generate_date, which the backend prefers, and a reason only
364425
applies to an empty build, whose payload is far below the size cap.
426+
427+
The requester keys go on after the metrics, so a metric that happened to
428+
share their name could not replace who asked for the build.
365429
"""
366430
futures = []
367431
failed = 0
@@ -378,6 +442,8 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc
378442
if provenance:
379443
payload.update(provenance)
380444
payload.update(d["metrics"])
445+
if requester:
446+
payload.update(requester)
381447

382448
message_data = json.dumps(payload, default=str).encode("utf-8")
383449
try:
@@ -477,8 +543,9 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc
477543
if reason_problem:
478544
print(f"[WARN] Ignoring --noMetricsReason because {reason_problem}.")
479545
provenance = load_provenance(args.provenanceFile)
546+
requester = resolve_requester(args)
480547
payload = build_pipeline_payload(
481-
design_records, args, provenance, no_metrics_reason
548+
design_records, args, provenance, no_metrics_reason, requester
482549
)
483550
message_data = json.dumps(payload, default=str).encode("utf-8")
484551

@@ -494,7 +561,7 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc
494561
"backend will report this build's completeness as unknown."
495562
)
496563
failed = publish_v1_per_design(
497-
publisher, topic_path, design_records, args, provenance
564+
publisher, topic_path, design_records, args, provenance, requester
498565
)
499566
if failed:
500567
print(

0 commit comments

Comments
 (0)