diff --git a/flow/test/test_uploadMetadata.py b/flow/test/test_uploadMetadata.py index ec234f6194..92555ed5a5 100755 --- a/flow/test/test_uploadMetadata.py +++ b/flow/test/test_uploadMetadata.py @@ -61,6 +61,12 @@ def from_service_account_file(path): BUILD_TIME_FORMAT = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$") +REQUESTER_ENV = ("UPLOAD_BUILD_USER_EMAIL", "UPLOAD_CHANGE_AUTHOR_EMAIL") + +# Large enough to push the pipeline payload over the 8 MB cap, so the script +# falls back to one v1 message per design. +OVERSIZED_METRICS = {"blob": "x" * (9 * 1024 * 1024)} + class TestUploadMetadata(unittest.TestCase): def setUp(self): @@ -96,8 +102,11 @@ def add_design(self, platform, design, variant="base", metrics=None): with open(os.path.join(report_dir, "metadata.json"), "w") as f: json.dump(metrics or {"finish:area": 1.0}, f) - def run_script(self, *extra, fail=False, tz=None): + def run_script(self, *extra, fail=False, tz=None, extra_env=None): env = dict(os.environ) + for name in REQUESTER_ENV: + env.pop(name, None) + env.update(extra_env or {}) if tz: env["TZ"] = tz env["PYTHONPATH"] = self.stubs @@ -343,6 +352,178 @@ def test_empty_designs_valid_provenance_is_v4(self): self.assertEqual(payload["designs"], []) self.assertEqual(payload["jenkins_library"], {"ref": "v1.0.0"}) + def test_pipeline_payload_without_requester_is_unchanged(self): + self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0}) + result, messages = self.run_script("--jobName", "ORFS") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertRegex(payload.pop("build_time"), BUILD_TIME_FORMAT) + self.assertEqual( + payload, + { + "payload_schema_version": 3, + "jenkins_env": "public", + "build_id": "42", + "branch_name": "master", + "pipeline_id": "jenkins-ORFS-master-42", + "change_branch": None, + "commit_sha": "abc123", + "jenkins_url": "http://jenkins/job/ORFS/42/", + "job_name": "ORFS", + "designs": [ + { + "platform": "nangate45", + "design": "gcd", + "variant": "base", + "metrics": {"finish__area": 5.0}, + } + ], + }, + ) + + def test_v1_message_without_requester_is_unchanged(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + result, messages = self.run_script() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(messages), 1) + payload = messages[0]["data"] + self.assertEqual(payload.pop("blob"), OVERSIZED_METRICS["blob"]) + self.assertEqual( + payload, + { + "build_id": "42", + "branch_name": "master", + "pipeline_id": "jenkins-ORFS-master-42", + "change_branch": None, + "commit_sha": "abc123", + "jenkins_url": "http://jenkins/job/ORFS/42/", + "jenkins_env": "public", + }, + ) + + def test_both_requester_flags_reach_pipeline_payload(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + "--buildUserEmail", + " ada@example.com ", + "--changeAuthorEmail", + "bob@example.com", + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertEqual(payload["build_user_email"], "ada@example.com") + self.assertEqual(payload["change_author_email"], "bob@example.com") + self.assertEqual(payload["payload_schema_version"], 2) + + def test_both_requester_flags_reach_v1_messages(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + result, messages = self.run_script( + "--buildUserEmail", + "ada@example.com", + "--changeAuthorEmail", + "bob@example.com", + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertEqual(payload["build_user_email"], "ada@example.com") + self.assertEqual(payload["change_author_email"], "bob@example.com") + + def test_blank_requester_flag_is_omitted(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + "--buildUserEmail", " ", "--changeAuthorEmail", "bob@example.com" + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertNotIn("build_user_email", payload) + self.assertEqual(payload["change_author_email"], "bob@example.com") + + def test_blank_requester_flag_is_omitted_from_v1_messages(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + result, messages = self.run_script( + "--buildUserEmail", "ada@example.com", "--changeAuthorEmail", "" + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertEqual(payload["build_user_email"], "ada@example.com") + self.assertNotIn("change_author_email", payload) + + def test_requester_without_at_sign_is_dropped(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + "--buildUserEmail", "null", "--changeAuthorEmail", "None" + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertNotIn("build_user_email", payload) + self.assertNotIn("change_author_email", payload) + self.assertIn("[WARN] Not sending --buildUserEmail: 'null'", result.stdout) + self.assertIn("[WARN] Not sending --changeAuthorEmail: 'None'", result.stdout) + + def test_v1_fallback_warns_about_requester_once(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + result, messages = self.run_script("--buildUserEmail", "null") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("build_user_email", messages[0]["data"]) + self.assertEqual(result.stdout.count("Not sending --buildUserEmail"), 1) + + def test_requester_is_read_from_environment(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + extra_env={ + "UPLOAD_BUILD_USER_EMAIL": "ada@example.com", + "UPLOAD_CHANGE_AUTHOR_EMAIL": " bob@example.com ", + } + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + payload = messages[0]["data"] + self.assertEqual(payload["build_user_email"], "ada@example.com") + self.assertEqual(payload["change_author_email"], "bob@example.com") + + def test_requester_flag_overrides_environment(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + "--buildUserEmail", + "ada@example.com", + extra_env={"UPLOAD_BUILD_USER_EMAIL": "other@example.com"}, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(messages[0]["data"]["build_user_email"], "ada@example.com") + + def test_empty_requester_flag_suppresses_environment(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + "--buildUserEmail", + "", + extra_env={"UPLOAD_BUILD_USER_EMAIL": "ada@example.com"}, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("build_user_email", messages[0]["data"]) + + def test_environment_requester_is_validated(self): + self.add_design("nangate45", "gcd") + result, messages = self.run_script( + extra_env={"UPLOAD_BUILD_USER_EMAIL": "null"} + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("build_user_email", messages[0]["data"]) + self.assertIn("[WARN] Not sending --buildUserEmail: 'null'", result.stdout) + + def test_environment_requester_reaches_v1_messages(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + result, messages = self.run_script( + extra_env={"UPLOAD_CHANGE_AUTHOR_EMAIL": "bob@example.com"} + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(messages[0]["data"]["change_author_email"], "bob@example.com") + + def test_requester_keys_win_over_same_named_v1_metric(self): + metrics = dict(OVERSIZED_METRICS, build_user_email="metric") + self.add_design("nangate45", "gcd", metrics=metrics) + result, messages = self.run_script("--buildUserEmail", "ada@example.com") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(messages[0]["data"]["build_user_email"], "ada@example.com") + def test_no_publisher_publishes_nothing(self): env = dict(os.environ) env["PYTHONPATH"] = self.stubs diff --git a/flow/util/uploadMetadata.py b/flow/util/uploadMetadata.py index bf74fac9e0..5aff02839a 100755 --- a/flow/util/uploadMetadata.py +++ b/flow/util/uploadMetadata.py @@ -111,6 +111,24 @@ def _non_negative_int(val): "a warning). Emitted as no_metrics_reason only on a payload with no " "designs.", ) +parser.add_argument( + "--buildUserEmail", + type=str, + default=None, + help="Email of the Jenkins user who started the build by hand " + "(BUILD_USER_EMAIL). Omitted for SCM, timer and branch-scan triggers. " + "The dashboard attributes the build to this user. Defaults to " + "$UPLOAD_BUILD_USER_EMAIL, which keeps the address out of a traced " + "(sh -x) command line.", +) +parser.add_argument( + "--changeAuthorEmail", + type=str, + default=None, + help="Email of the pull request's author on a PR build " + "(CHANGE_AUTHOR_EMAIL). Empty when the author's GitHub email is private. " + "Defaults to $UPLOAD_CHANGE_AUTHOR_EMAIL.", +) # --- PUBSUB args --- parser.add_argument( @@ -253,6 +271,45 @@ def resolve_schema_version(args, provenance): return 4 if provenance else 3 +def resolve_requester(args): + """Who asked for the build, as the payload keys the backend reads. + + Each value comes from its flag or, when the flag is absent, from its + environment variable. An empty flag counts as given, so it suppresses the + variable. Blank values are left out rather than sent empty, and a value + without an "@" is dropped with a warning: an unset Groovy variable + interpolates as the string "null", which must not be stored as an email. + The keys ride on every schema version: the backend reads them by presence, + and they describe the build rather than the payload's shape, so they do + not bump the version. + """ + requester = {} + for key, flag, value, env_var in ( + ( + "build_user_email", + "--buildUserEmail", + args.buildUserEmail, + "UPLOAD_BUILD_USER_EMAIL", + ), + ( + "change_author_email", + "--changeAuthorEmail", + args.changeAuthorEmail, + "UPLOAD_CHANGE_AUTHOR_EMAIL", + ), + ): + if value is None: + value = os.environ.get(env_var, "") + value = value.strip() + if not value: + continue + if "@" not in value: + print(f"[WARN] Not sending {flag}: '{value}' is not an email address.") + continue + requester[key] = value + return requester + + def resolve_build_time(args): """The build time to report, defaulting to now in genMetrics.py's format. @@ -283,7 +340,7 @@ def resolve_no_metrics_reason(args, design_records): def build_pipeline_payload( - design_records, args, provenance=None, no_metrics_reason=None + design_records, args, provenance=None, no_metrics_reason=None, requester=None ): """Return the pipeline-level payload dict. @@ -316,6 +373,8 @@ def build_pipeline_payload( # the payload. if schema_version == 4: payload.update(provenance) + if requester: + payload.update(requester) return payload @@ -339,7 +398,9 @@ def publish_pipeline_report( print(f"[INFO] Published pipeline report to Pub/Sub (message ID: {message_id}).") -def publish_v1_per_design(publisher, topic_path, design_records, args, provenance=None): +def publish_v1_per_design( + publisher, topic_path, design_records, args, provenance=None, requester=None +): """Fallback path used when the v2 pipeline payload exceeds MAX_PUBSUB_BYTES. Emits one v1-format message per design (no payload_schema_version, metrics @@ -362,6 +423,9 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc metadata there but not needed: each design's metrics carry run__flow__generate_date, which the backend prefers, and a reason only applies to an empty build, whose payload is far below the size cap. + + The requester keys go on after the metrics, so a metric that happened to + share their name could not replace who asked for the build. """ futures = [] failed = 0 @@ -378,6 +442,8 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc if provenance: payload.update(provenance) payload.update(d["metrics"]) + if requester: + payload.update(requester) message_data = json.dumps(payload, default=str).encode("utf-8") try: @@ -477,8 +543,9 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc if reason_problem: print(f"[WARN] Ignoring --noMetricsReason because {reason_problem}.") provenance = load_provenance(args.provenanceFile) + requester = resolve_requester(args) payload = build_pipeline_payload( - design_records, args, provenance, no_metrics_reason + design_records, args, provenance, no_metrics_reason, requester ) message_data = json.dumps(payload, default=str).encode("utf-8") @@ -494,7 +561,7 @@ def publish_v1_per_design(publisher, topic_path, design_records, args, provenanc "backend will report this build's completeness as unknown." ) failed = publish_v1_per_design( - publisher, topic_path, design_records, args, provenance + publisher, topic_path, design_records, args, provenance, requester ) if failed: print(