44(https://dashboard.precisioninno.com).
55
66The check POSTs a run's numeric metrics from metadata.json to the dashboard,
7- which compares them against a real baseline build -- the latest master build
8- by default -- using the thresholds it stores in rule_configs. This is the QoR
7+ which compares them against a real baseline build -- the master build at the
8+ run's merge base, by default -- using the thresholds it stores in rule_configs. This is the QoR
99gate behind `make metadata`. Rule tolerances live in the dashboard, not in
1010this repository, and the baseline moves with master, so a merged change
1111becomes the new baseline without a file update.
2121 export DASHBOARD_API_KEY=plt_... # optional; needed for private platforms
2222 ./util/checkQorMetrics.py
2323
24+ Each run is checked as the commit metadata.json records (--commit overrides
25+ it). The dashboard searches its default baseline pipelines in order and takes
26+ a master build that has the design: the one at the commit's merge base with
27+ master, else the latest one before the commit. A commit on master is its own
28+ merge base, so a master run is compared against that commit's own build.
29+ Without a commit the run is compared against the latest master build.
30+
31+ --job-name (DASHBOARD_JOB_NAME) checks as a build of that pipeline instead, with
32+ its CI rules for the branch checked out here (--branch overrides it): the
33+ previous build on a trunk branch, the merge base on any other branch. CI
34+ passes --branch and --commit (or DASHBOARD_BRANCH and DASHBOARD_COMMIT), since
35+ its checkout has no branch.
36+
2437Exit status: 1 when the dashboard reports a failed rule for any checked run,
2538or when a metadata.json is missing, malformed, or holds no numeric metric.
26390 otherwise. A run the dashboard could not judge -- no baseline, nothing
3447import argparse
3548import concurrent .futures
3649import json
50+ import math
3751import os
3852import re
53+ import subprocess
3954import sys
4055import time
4156import urllib .error
6580RETRY_BACKOFF_SECONDS = 1.0
6681RETRYABLE_STATUS = frozenset ({429 , 500 , 502 , 503 , 504 })
6782
68- DEFAULT_JOB_NAME = "OpenROAD-flow-scripts-Public"
69-
7083# metadata.json records the ORFS commit, but genMetrics.py writes the strings
7184# "git not on PATH" / "not a git repo" / "N/A" there when it cannot resolve one.
7285SHA_RE = re .compile (r"^[0-9a-f]{40}$" )
@@ -148,7 +161,9 @@ class MetadataError(Exception):
148161
149162
150163def load_metadata (path , only_prefix = None ):
151- """Returns (metrics, commit_sha). Raises MetadataError if the file is unusable.
164+ """Returns (metrics, commit_sha, dropped). Raises MetadataError if the file is unusable.
165+
166+ dropped names the metrics left out because their value is not finite.
152167
153168 only_prefix keeps just the metrics whose name starts with one of the given
154169 prefixes. The dashboard evaluates only the metrics it receives, so sending
@@ -180,6 +195,14 @@ def load_metadata(path, only_prefix=None):
180195 # reported metric count honest.
181196 metrics = {key : value for key , value in metrics .items () if "__iter__" not in key }
182197
198+ # ORFS writes inf/-inf/NaN for a metric it could not compute (an unconstrained
199+ # timing path reports -inf TNS). json.dumps renders those as bare Infinity/NaN
200+ # tokens, which are not JSON: the dashboard rejects the whole request with
201+ # HTTP 400, so one unusable metric would cost the run its verdict. A value
202+ # that is not finite has nothing to compare against, so leave it out.
203+ dropped = sorted (key for key , value in metrics .items () if not math .isfinite (value ))
204+ metrics = {key : value for key , value in metrics .items () if key not in dropped }
205+
183206 if only_prefix :
184207 metrics = {
185208 key : value
@@ -191,7 +214,27 @@ def load_metadata(path, only_prefix=None):
191214 raise MetadataError (f"no numeric metrics in { path } " )
192215
193216 commit = str (raw .get (COMMIT_KEY , "" ))
194- return metrics , commit if SHA_RE .match (commit ) else ""
217+ return metrics , commit if SHA_RE .match (commit ) else "" , dropped
218+
219+
220+ def current_branch ():
221+ """The branch checked out in this repository, or "" when git cannot say.
222+
223+ A detached HEAD reads as "HEAD", which the dashboard treats as any other
224+ non-trunk branch: it compares against the merge base, which is the right
225+ baseline for a detached checkout too.
226+ """
227+ try :
228+ output = subprocess .run (
229+ ["git" , "rev-parse" , "--abbrev-ref" , "HEAD" ],
230+ capture_output = True ,
231+ text = True ,
232+ timeout = 10 ,
233+ check = True ,
234+ ).stdout
235+ except (OSError , subprocess .SubprocessError ):
236+ return ""
237+ return output .strip ()
195238
196239
197240def post_check (endpoint , api_key , payload ):
@@ -234,8 +277,22 @@ def post_check(endpoint, api_key, payload):
234277 return None , f"{ error_message } (after { RETRY_ATTEMPTS } attempts)"
235278
236279
237- def check_run (run , endpoint , api_key , base_commit , job_name , only_prefix = None ):
238- """Checks one (platform, design, variant) and returns a result dict."""
280+ def check_run (
281+ run ,
282+ endpoint ,
283+ api_key ,
284+ base_commit ,
285+ job_name ,
286+ only_prefix = None ,
287+ commit = "" ,
288+ branch = "" ,
289+ ):
290+ """Checks one (platform, design, variant) and returns a result dict.
291+
292+ commit replaces the one metadata.json records. branch is sent only with a
293+ job_name: without one the dashboard ignores it and resolves the baseline
294+ from the commit alone.
295+ """
239296 platform , design , variant , metadata_path = run
240297 result = {
241298 "platform" : platform ,
@@ -245,6 +302,7 @@ def check_run(run, endpoint, api_key, base_commit, job_name, only_prefix=None):
245302 "response" : None ,
246303 "error" : None ,
247304 "metric_count" : 0 ,
305+ "dropped" : [],
248306 }
249307
250308 # One run's surprise must not take down the sweep: this is called from a
@@ -255,24 +313,33 @@ def check_run(run, endpoint, api_key, base_commit, job_name, only_prefix=None):
255313 # A bad local file is the caller's problem, not the service's: it gets
256314 # its own status so it exits 1 instead of passing as inconclusive.
257315 try :
258- metrics , commit = load_metadata (metadata_path , only_prefix )
316+ metrics , recorded_commit , dropped = load_metadata (
317+ metadata_path , only_prefix
318+ )
259319 except MetadataError as error :
260320 result ["status" ] = "invalid"
261321 result ["error" ] = str (error )
262322 return result
263323 result ["metric_count" ] = len (metrics )
264-
324+ result ["dropped" ] = dropped
325+ commit = commit or recorded_commit
326+ result ["commit" ] = commit
327+
328+ # No JobName and no PipelineId makes this a checkout to the dashboard:
329+ # it picks the pipeline itself, the first of its default baseline
330+ # pipelines with a build of this design. A JobName makes it a build of
331+ # that pipeline, resolved by that pipeline's rules for BranchName.
265332 payload = {
266333 "CommitSha" : commit ,
267- "BranchName" : "" ,
334+ "BranchName" : branch if job_name else "" ,
268335 "PipelineId" : "" ,
269- "JobName" : job_name ,
270336 "PlatformName" : platform ,
271337 "DesignName" : design ,
272338 "VariantName" : variant ,
273339 "Metrics" : metrics ,
274- "IsLocal" : True ,
275340 }
341+ if job_name :
342+ payload ["JobName" ] = job_name
276343 if base_commit :
277344 payload ["BaseCommitSha" ] = base_commit
278345
@@ -334,10 +401,22 @@ def render_run(result, verbose):
334401 f"{ passed } passed, { failed } failed, { missing } missing "
335402 f"({ result ['metric_count' ]} metrics sent)"
336403 )
404+ pipeline = response .get ("baselinePipeline" )
337405 lines .append (
338406 f"[INFO] baseline build { response .get ('baseBuildId' )} "
339- f"({ (response .get ('baseCommitSha' ) or '?' )[:12 ]} , { strategy } )"
407+ f"({ (response .get ('baseCommitSha' ) or '?' )[:12 ]} , { strategy } "
408+ f"{ ', ' + pipeline if pipeline else '' } )"
340409 )
410+ if not result .get ("commit" ) and strategy == "latest-master" :
411+ lines .append (
412+ "[WARN] metadata.json records no commit, so this was compared "
413+ "against the latest master build."
414+ )
415+ if result ["dropped" ]:
416+ lines .append (
417+ f"[WARN] Left out { len (result ['dropped' ])} non-finite metric(s): "
418+ + ", " .join (result ["dropped" ])
419+ )
341420
342421 if result ["status" ] == "inconclusive" :
343422 lines .append (
@@ -501,15 +580,34 @@ def parse_args(argv):
501580 )
502581 parser .add_argument (
503582 "--base-commit" ,
504- help = "Pin the baseline to this commit's build instead of latest master." ,
583+ help = "Pin the baseline to this commit's build instead of resolving one." ,
584+ )
585+ parser .add_argument (
586+ "--api-url" ,
587+ default = os .environ .get ("DASHBOARD_API_URL" ) or None ,
588+ help = "Dashboard API base URL, e.g. a staging deployment. Takes "
589+ "precedence over --beta. Env: DASHBOARD_API_URL." ,
590+ )
591+ parser .add_argument (
592+ "--commit" ,
593+ default = os .environ .get ("DASHBOARD_COMMIT" , "" ),
594+ help = "Commit to check as, instead of the one metadata.json records. "
595+ "Env: DASHBOARD_COMMIT." ,
596+ )
597+ parser .add_argument (
598+ "--branch" ,
599+ default = os .environ .get ("DASHBOARD_BRANCH" ) or None ,
600+ help = "With --job-name: the branch to check as, instead of the one "
601+ "checked out here, e.g. the PR branch in CI. "
602+ "Env: DASHBOARD_BRANCH." ,
505603 )
506604 parser .add_argument (
507605 "--job-name" ,
508- default = os .environ .get ("DASHBOARD_JOB_NAME" , DEFAULT_JOB_NAME ) ,
509- help = "Canonical Jenkins job-folder name the dashboard uses to scope the master "
510- f"baseline to a pipeline . Default: { DEFAULT_JOB_NAME } . Override for a "
511- "secure/private branch whose master-equivalent builds elsewhere, e.g . "
512- "OpenROAD-flow-scripts-Private ." ,
606+ default = os .environ .get ("DASHBOARD_JOB_NAME" ) or None ,
607+ help = "Check as a build of this CI pipeline, with its rules for the "
608+ "branch . Default: the dashboard searches its default baseline "
609+ "pipelines and takes the first with a build of the design . "
610+ "Env: DASHBOARD_JOB_NAME ." ,
513611 )
514612 parser .add_argument (
515613 "--metadata" ,
@@ -612,8 +710,16 @@ def selected(entry):
612710def run_check (argv = None ):
613711 args = parse_args (argv )
614712
615- endpoint = (BETA_API_URL if args .beta else API_URL ) + CHECK_PATH
713+ api_url = args .api_url or (BETA_API_URL if args .beta else API_URL )
714+ endpoint = api_url .rstrip ("/" ) + CHECK_PATH
616715 api_key = os .environ .get ("DASHBOARD_API_KEY" , "" )
716+ # Strip anything but hex digits: the commit reaches the dashboard, and a
717+ # caller may pass one with a trailing newline or a ref prefix.
718+ commit = re .sub (r"[^0-9a-fA-F]" , "" , args .commit )
719+ # Only a pipeline run uses the branch; a checkout is resolved from the commit.
720+ branch = ""
721+ if args .job_name :
722+ branch = (args .branch if args .branch is not None else current_branch ()).strip ()
617723
618724 if args .metadata :
619725 # Single-run mode: the caller (make metadata-check) names the run and
@@ -642,6 +748,21 @@ def run_check(argv=None):
642748 )
643749 if args .base_commit :
644750 header .append (f"[INFO] Baseline pinned to commit { args .base_commit } " )
751+ elif args .job_name and branch :
752+ header .append (
753+ f"[INFO] Checking as a build of { args .job_name } , branch { branch } "
754+ + (f", commit { commit } " if commit else "" )
755+ )
756+ elif args .job_name :
757+ header .append (
758+ f"[WARN] Checking as a build of { args .job_name } with no branch (git "
759+ "could not name one, and no --branch): resolved as a feature branch."
760+ )
761+ else :
762+ header .append (
763+ "[INFO] Baseline from the dashboard's default pipelines, resolved "
764+ "from " + (f"commit { commit } " if commit else "each run's recorded commit" )
765+ )
645766 if args .only_prefix :
646767 prefixes = " " .join (args .only_prefix )
647768 header .append (f"[INFO] Sending only metrics prefixed { prefixes } " )
@@ -661,6 +782,8 @@ def run_check(argv=None):
661782 args .base_commit ,
662783 args .job_name ,
663784 args .only_prefix ,
785+ commit ,
786+ branch ,
664787 ),
665788 runs ,
666789 )
0 commit comments