diff --git a/examples/remote_management/utilization_report.py b/examples/remote_management/utilization_report.py new file mode 100644 index 000000000..f353950cf --- /dev/null +++ b/examples/remote_management/utilization_report.py @@ -0,0 +1,1550 @@ +"""Flyte v2 usage metering task with an interactive HTML report. + +Sweeps every project/domain/run/action reachable on the backend, records +requested CPU/memory/GPU and per-phase durations for each action (including +traces and plugin tasks), and renders an interactive report (flyte.report) +with: + + - filters: date range, project, domain, user, task type + - group-by: project / domain / user / run / task / task type, bucketed by + day / month / year + - editable assumptions: default requests for tasks that declared none + (k8s admission defaults are not visible via the Flyte API) + - drill-down: group -> runs -> full action tree (sub-actions and traces) + - fanout aggregation: identical leaf actions under one parent collapse into + a single counted row (metrics stay exact; memory and report size stay + bounded even for 200k-wide fanouts) + - crash recovery: the run list and each completed 50-run sweep batch are + persisted to a flyte.Checkpoint at a stable URI, restored on platform + retries AND on the bigger-container re-run the report task launches if + the sweep OOMs — completed work is never redone + +Run it (the report appears in the Flyte UI): + flyte run utilization_report.py usage_report [--start YYYY-MM-DD] [--end YYYY-MM-DD] + [--project P] [--domain D] [--all-scopes] +or with raw python (init from a config, then submit the same task): + python utilization_report.py --config ~/.union/config.yaml [--all] [--start ...] + +Default scope is the project/domain the task runs in; default window is the +last 12 full months plus the current month. +""" + +import argparse +import asyncio +import json +import os +import re +import sys + +import flyte +import flyte.errors +import flyte.report +from flyte.io import File + +env = flyte.TaskEnvironment( + name="usage_profiler", + resources=flyte.Resources(cpu=2, memory="2Gi"), +) + +CONCURRENCY = 24 # concurrent action-detail RPCs per batch +RUN_CONCURRENCY = 8 # concurrent runs being processed per batch +BATCH_RUNS = 50 # runs per sweep batch (checkpoint granularity) +BATCH_CONCURRENCY = 4 # sweep batches in flight at once (traces gather safely) +SCOPE_CONCURRENCY = 8 # project/domain run listings in flight at once +RUNS_PAGE = 500 # list_runs page size +ACTIONS_PAGE = 1000 # list_actions page size — the server honors large pages +# at ~the same per-page latency as 100, so this cuts a 200k-action run from +# 2000 sequential round trips to 200 +OOM_RETRY_RESOURCES = flyte.Resources(cpu=2, memory="8Gi") +CHECKPOINT_SAVE_INTERVAL = 30.0 # seconds between checkpoint blob rewrites + +_MEM_UNITS = { + "": 1, + "k": 10**3, + "M": 10**6, + "G": 10**9, + "T": 10**12, + "P": 10**15, + "Ki": 2**10, + "Mi": 2**20, + "Gi": 2**30, + "Ti": 2**40, + "Pi": 2**50, +} + + +def parse_cpu(v: str) -> float: + if not v: + return 0.0 + v = v.strip() + if v.endswith("m"): + return float(v[:-1]) / 1000 + return float(v) + + +def parse_mem_gib(v: str) -> float: + if not v: + return 0.0 + m = re.fullmatch(r"([0-9.]+)\s*([A-Za-z]*)", v.strip()) + if not m: + return 0.0 + return float(m.group(1)) * _MEM_UNITS.get(m.group(2), 1) / 2**30 + + +def extract_resources(detail_dict: dict) -> dict: + """Requested resources from the action's task template. + + cpu/mem are None when the task declared no request (k8s namespace + defaults apply at admission and are invisible to the Flyte API) — + the report substitutes editable assumed values for those. + """ + tmpl = (detail_dict.get("task") or {}).get("taskTemplate") or {} + container = tmpl.get("container") or {} + out = { + "cpu": None, + "mem": None, + "gpu": 0.0, + "gd": "", + "hc": bool(container), + "tmpl_type": str(tmpl.get("type", "")), + "has_tmpl": bool(tmpl), + } + for r in (container.get("resources") or {}).get("requests") or []: + name, value = r.get("name"), r.get("value", "") + if name == "CPU": + out["cpu"] = parse_cpu(value) + elif name == "MEMORY": + out["mem"] = parse_mem_gib(value) + elif name == "GPU": + out["gpu"] = float(value or 0) + acc = (tmpl.get("extendedResources") or {}).get("gpuAccelerator") or {} + out["gd"] = acc.get("device", "") + return out + + +def resolve_window(start: str = "", end: str = ""): + """Parse YYYY-MM-DD bounds; default = last 12 full months + current month.""" + from datetime import datetime, timezone + + now = datetime.now(timezone.utc) + if start: + start_dt = datetime.fromisoformat(start).replace(tzinfo=timezone.utc) + else: + y, m = now.year, now.month - 12 + if m <= 0: + y, m = y - 1, m + 12 + start_dt = datetime(y, m, 1, tzinfo=timezone.utc) + end_dt = datetime.fromisoformat(end).replace(tzinfo=timezone.utc) if end else None + return start_dt, end_dt + + +async def load_scopes(project: str = "", domain: str = "") -> list[list[str]]: + """Project/domain pairs to sweep.""" + from flyte.remote import Project + + scopes: list[list[str]] = [] + async for p in Project.listall.aio(): + pd = p.to_dict() + pname = str(pd.get("id") or pd.get("name") or "") + if not pname or (project and pname != project): + continue + for d in pd.get("domains") or []: + dname = str(d.get("id") or "") + if not dname or (domain and dname != domain): + continue + scopes.append([pname, dname]) + return scopes + + +async def load_runs(scopes: list[list[str]], start: str = "", end: str = "") -> dict: + """All in-window runs across the scopes, plus resolved user names. + + start/end are ISO dates bounding by run start time (run listings are + newest-first, so pages older than `start` are never fetched). Called only + when there is no checkpoint yet: the result is pinned in the checkpoint + blob so every restart sweeps exactly the same runs. + """ + from datetime import datetime, timezone + + from flyteidl2.common import identifier_pb2, list_pb2 + from flyteidl2.workflow import run_service_pb2 + + from flyte._initialize import get_client, get_init_config + from flyte.remote import User + + org = get_init_config().org + cutoff = datetime.fromisoformat(start).replace(tzinfo=timezone.utc) if start else None + end_dt = datetime.fromisoformat(end).replace(tzinfo=timezone.utc) if end else None + + user_names: dict[str, str] = {} + try: + me = await User.get.aio() + subj = me.subject() if callable(me.subject) else me.subject + name = me.name() if callable(me.name) else me.name + if subj: + user_names[str(subj)] = str(name or subj) + except Exception: + pass + + # List runs with the raw client: the SDK's Run.listall pagination stalls + # after the first page (re-yields it), silently dropping older runs. + # Scopes are independent, so they list concurrently; pagination within a + # scope stays sequential (token chain). Results merge in scope order. + scope_sem = asyncio.Semaphore(SCOPE_CONCURRENCY) + + async def list_scope(proj: str, dom: str) -> list[list[str]]: + scope_runs: list[list[str]] = [] + seen: set[str] = set() + token, pages = None, 0 + async with scope_sem: + try: + while True: + # timeout + repeated-token guard: a hung or looping page + # must not stall the whole sweep silently + resp = await asyncio.wait_for( + get_client().run_service.list_runs( + run_service_pb2.ListRunsRequest( + request=list_pb2.ListRequest(limit=RUNS_PAGE, token=token or ""), + org=org, + project_id=identifier_pb2.ProjectIdentifier(organization=org, domain=dom, name=proj), + ) + ), + 60, + ) + pages += 1 + if pages % 20 == 0: + print(f" {proj}/{dom}: page {pages}, {len(scope_runs)} runs kept …", flush=True) + page_all_old = bool(resp.runs) and cutoff is not None + for r in resp.runs: + if r.action.status.HasField("start_time"): + started = r.action.status.start_time.ToDatetime(tzinfo=timezone.utc) + if cutoff is not None and started < cutoff: + continue + page_all_old = False + if end_dt is not None and started > end_dt: + continue + else: + page_all_old = False + if r.action.id.run.name in seen: + continue + seen.add(r.action.id.run.name) + eb = r.action.metadata.executed_by + user = eb.user.id.subject or eb.application.id.subject or "unknown" + # run protos carry the user's profile — resolve names for everyone + sp = eb.user.spec + label = f"{sp.first_name} {sp.last_name}".strip() or sp.email or sp.user_handle + if eb.user.id.subject and label: + user_names.setdefault(eb.user.id.subject, label) + elif eb.application.id.subject and eb.application.spec.name: + user_names.setdefault(eb.application.id.subject, eb.application.spec.name) + scope_runs.append([proj, dom, r.action.id.run.name, user]) + if resp.token and resp.token == token: + print(f" ! {proj}/{dom}: server repeated page token, stopping this scope", file=sys.stderr) + break + token = resp.token + # newest-first: once a whole page is older than the cutoff, stop + if not token or page_all_old: + break + except Exception as e: + print(f" ! listing runs {proj}/{dom}: {type(e).__name__}: {e}", file=sys.stderr) + print(f" {proj}/{dom}: {len(scope_runs)} runs", flush=True) + return scope_runs + + per_scope = await asyncio.gather(*(list_scope(proj, dom) for proj, dom in scopes)) + runs: list[list[str]] = [r for scope_runs in per_scope for r in scope_runs] + return {"runs": runs, "user_names": user_names} + + +def aggregate_fanout(out: list[dict]) -> list[dict]: + """Collapse identical leaf actions under one parent into a counted row. + + Only actions that are nobody's parent are aggregated — the run's full + action set is listed before this is called, so "has children" is decidable + from the rows' parent links. A 200k-wide fanout of one task collapses to a + single row carrying `cnt` and summed durations (day granularity is part of + the group key), so every metric stays exact while memory and report size + stay bounded. Anything with children keeps its own row for drill-down. + """ + parents = {r["pa"] for r in out if r["pa"]} + kept: list[dict] = [] + groups: dict[tuple, dict] = {} + for r in out: + if r["an"] in parents: + kept.append(r) + continue + key = ( + r["pa"], + r["at"], + r["tt"], + r["tn"], + r["ph"], + r["cs"], + r["us"], + r["cpu"], + r["mem"], + r["gpu"], + r["gd"], + r["hc"], + r["rs"] > 0, + (r["st"] or "")[:10], + ) + g = groups.get(key) + if g is None: + groups[key] = {**r, "cnt": 1} + else: + g["cnt"] += 1 + for f in ("qs", "ins", "rs", "ts"): + g[f] += r[f] + g["att"] = max(g["att"], r["att"]) + if r["st"] and (not g["st"] or r["st"] < g["st"]): + g["st"] = r["st"] + return kept + list(groups.values()) + + +async def sweep_batch(index: int, total: int, batch: list[list[str]]) -> list[dict]: + """Sweep one batch of runs into fanout-aggregated report rows. + + Each completed batch's rows are persisted to the caller's checkpoint + blob, so a restart only re-sweeps batches that were still in flight. + """ + import time + + from flyteidl2.common import identifier_pb2, list_pb2, phase_pb2 + from flyteidl2.workflow import run_definition_pb2, run_service_pb2 + + from flyte._initialize import get_client, get_init_config + from flyte.remote._action import ActionDetails + + org = get_init_config().org + sem = asyncio.Semaphore(CONCURRENCY) + run_sem = asyncio.Semaphore(RUN_CONCURRENCY) + + try: + from flyteidl2.core import catalog_pb2 + + CACHE_HIT = catalog_pb2.CatalogCacheStatus.Value("CACHE_HIT") + except Exception: + CACHE_HIT = 2 # flyteidl2.core.CatalogCacheStatus.CACHE_HIT + + def proto_common(a) -> dict: + md, stt = a.metadata, a.status + atype = ( + run_definition_pb2.ActionType.Name(md.action_type).replace("ACTION_TYPE_", "") if md.action_type else "TASK" + ) + try: + phase = phase_pb2.ActionPhase.Name(stt.phase).replace("ACTION_PHASE_", "") + except Exception: + phase = str(stt.phase) + return { + "an": a.id.name, + "pa": md.parent, + "at": atype, + "tn": md.task.id.name, + "tt": md.task.task_type, + "ph": phase, + "att": stt.attempts, + "st": stt.start_time.ToJsonString() if stt.HasField("start_time") else "", + "cs": stt.cache_status == CACHE_HIT, + } + + def needs_details(c: dict) -> bool: + # traces, conditions, and engine-run orchestration primitives never run a + # container — the list proto already tells us everything billable. Cache + # hits skip details only when the listing already names their type. + if c["at"] != "TASK" or c["tt"].startswith("core-"): + return False + return not (c["cs"] and c["tt"]) + + RPC_TIMEOUT = 60 # a single hung RPC must not wedge the sweep + + stats = {"details": 0, "fast": 0} + + async def detail_row(proj, dom, run_name, user, common) -> dict | None: + ident = identifier_pb2.ActionIdentifier( + run=identifier_pb2.RunIdentifier(org=org, project=proj, domain=dom, name=run_name), + name=common["an"], + ) + async with sem: + try: + d = await asyncio.wait_for(ActionDetails.get_details.aio(ident), RPC_TIMEOUT) + except Exception as e: + print(f" ! {run_name}/{common['an']}: {type(e).__name__}: {e}", file=sys.stderr) + return None + stats["details"] += 1 + dd = d.to_dict() + meta = dd.get("metadata") or {} + status = dd.get("status") or {} + res = extract_resources(dd) + # some backends return skeleton list protos; prefer detail metadata + # (str() everywhere: unknown enum values dict-ify as ints on newer servers) + action_type = str(meta.get("actionType") or "").replace("ACTION_TYPE_", "") or common["at"] + # resolve the action type through every signal we have: detail metadata, + # the list proto, the task template's own type, and finally "function" + # for mapped/traced python functions (task id + funtionName, no template) + task_type = ( + str((meta.get("task") or {}).get("taskType", "")) + or common["tt"] + or res["tmpl_type"] + or ("function" if not res["has_tmpl"] and (meta.get("funtionName") or meta.get("task")) else "") + or (action_type.lower() if action_type != "TASK" else "") + ) + if action_type != "TASK" or task_type.startswith("core-"): + res["hc"] = False + if common["cs"] or status.get("cacheStatus") == "CACHE_HIT": + res["hc"] = False + + def secs(prop): + # phase-duration properties raise on UNSPECIFIED-phase actions + try: + td = getattr(d, prop) + return round(td.total_seconds(), 3) if td else 0.0 + except Exception: + return 0.0 + + total_s = secs("runtime") + if not total_s and status.get("durationMs"): + total_s = round(int(status["durationMs"]) / 1000, 3) + return { + "pj": proj, + "dm": dom, + "rn": run_name, + "an": common["an"], + "pa": meta.get("parent", "") or common["pa"], + "at": action_type, + "tt": task_type, + "tn": ((meta.get("task") or {}).get("id") or {}).get("name", "") or common["tn"], + "ph": str(status.get("phase") or "").replace("ACTION_PHASE_", "") or common["ph"], + "att": status.get("attempts", 0) or common["att"], + "st": status.get("startTime", "") or common["st"], + "cs": common["cs"] or (status.get("cacheStatus") == "CACHE_HIT"), + "qs": secs("queued_time"), + "ins": secs("initializing_time"), + "rs": secs("running_time"), + "ts": total_s, + "cpu": res["cpu"], + "mem": res["mem"], + "gpu": res["gpu"], + "gd": res["gd"], + "hc": res["hc"], + "us": user, + } + + def fast_row(proj, dom, run_name, user, a, common) -> dict: + stats["fast"] += 1 + return { + "pj": proj, + "dm": dom, + "rn": run_name, + **common, + "tt": common["tt"] or (common["at"].lower() if common["at"] != "TASK" else ""), + "qs": 0.0, + "ins": 0.0, + "rs": 0.0, + "ts": round(a.status.duration_ms / 1000, 3) if a.status.duration_ms else 0.0, + "cpu": None, + "mem": None, + "gpu": 0.0, + "gd": "", + "hc": False, + "us": user, + } + + async def list_and_convert(proj: str, dom: str, run_name: str, user: str) -> tuple[list, list]: + """Page through a run's actions, converting each page to rows as it + arrives so the page's protos are dropped immediately — a 200k-action + run never holds its full proto list and its rows in memory at once. + Returns (pending detail commons, finished fast rows).""" + run_id = identifier_pb2.RunIdentifier(org=org, project=proj, domain=dom, name=run_name) + token = None + pending: list[dict] = [] + out: list[dict] = [] + while True: + req = list_pb2.ListRequest(limit=ACTIONS_PAGE, token=token) + resp = await asyncio.wait_for( + get_client().run_service.list_actions(run_service_pb2.ListActionsRequest(request=req, run_id=run_id)), + RPC_TIMEOUT, + ) + for a in resp.actions: + common = proto_common(a) + if needs_details(common): + pending.append(common) + else: + out.append(fast_row(proj, dom, run_name, user, a, common)) + if resp.token and resp.token == token: + print(f" ! {run_name}: server repeated action page token", file=sys.stderr) + break + token = resp.token + if not token: + break + return pending, out + + rows: list[dict] = [] + done = {"runs": 0} + t0 = time.monotonic() + + async def process_run(proj, dom, run_name, user): + async with run_sem: + try: + pending, out = await list_and_convert(proj, dom, run_name, user) + except Exception as e: + print(f" ! actions {run_name}: {e}", file=sys.stderr) + pending, out = [], [] + results = await asyncio.gather(*(detail_row(proj, dom, run_name, user, c) for c in pending)) + out.extend(r for r in results if r) + n_raw = len(out) + out = aggregate_fanout(out) + if n_raw > 1000: + print(f" aggregated {run_name}: {n_raw} actions -> {len(out)} rows", flush=True) + rows.extend(out) + done["runs"] += 1 + if done["runs"] % 20 == 0 or done["runs"] == len(batch): + el = time.monotonic() - t0 + print( + f" batch {index + 1}/{total}: {done['runs']}/{len(batch)} runs · {len(rows)} rows " + f"({stats['details']} detail RPCs, {stats['fast']} fast) · {el:.0f}s", + flush=True, + ) + + async def heartbeat(): + while True: + await asyncio.sleep(20) + el = time.monotonic() - t0 + print( + f" heartbeat: batch {index + 1}/{total} · {done['runs']}/{len(batch)} runs done · " + f"{len(rows)} rows · {stats['details']} detail RPCs · {el:.0f}s", + flush=True, + ) + + hb = asyncio.create_task(heartbeat()) + try: + await asyncio.gather(*(process_run(*r) for r in batch)) + finally: + hb.cancel() + return rows + + +@env.task(retries=1) +async def collect_actions( + start: str = "", end: str = "", project: str = "", domain: str = "", checkpoint_uri: str = "" +) -> File: + """Sweep runs/actions into fanout-aggregated report rows. + + Crash recovery via an explicit flyte.Checkpoint at `checkpoint_uri` — a + stable object-store URI the caller derives from its own output prefix. + The blob pins the run list (so every restart provably sweeps the same + runs) and holds each completed batch's rows. Because the URI is stable, + the same blob is restored on platform retries (retries=1 covers crashes + and other system errors) AND on the caller's bigger-memory re-run after + an OOM — unlike @flyte.trace or the platform's per-attempt checkpoint + paths, which only survive retries of the same action. OOM itself is + deliberately NOT handled here: same memory would just OOM again, so it + escalates to the caller. + """ + import time + + from flyte._initialize import get_init_config + + cp = flyte.Checkpoint(checkpoint_dest=checkpoint_uri, checkpoint_src=checkpoint_uri) if checkpoint_uri else None + state: dict = {} + if cp is not None: + payload = await cp.load() + if payload is not None and payload.is_file(): + try: + state = json.loads(payload.read_bytes()) + print(f"checkpoint restored: {len(state.get('batches', {}))} completed batches", flush=True) + except Exception as e: + print(f" ! unreadable checkpoint, starting fresh: {e}", file=sys.stderr) + state = {} + + if "runs" not in state: + scopes = await load_scopes(project=project, domain=domain) + print(f"scopes: {scopes}", flush=True) + listing = await load_runs(scopes=scopes, start=start, end=end) + state = {"runs": listing["runs"], "user_names": listing["user_names"], "batches": {}} + if cp is not None: + await cp.save(json.dumps(state).encode()) + + runs, user_names = state["runs"], state["user_names"] + done_batches: dict = state["batches"] # batch index (as str, for JSON) -> rows + print(f"runs to sweep: {len(runs)}", flush=True) + + batches = [runs[i : i + BATCH_RUNS] for i in range(0, len(runs), BATCH_RUNS)] + batch_sem = asyncio.Semaphore(BATCH_CONCURRENCY) + save_lock = asyncio.Lock() # serialize state mutation + blob rewrite + last_save = {"t": time.monotonic()} + + async def run_batch(i: int, b: list[list[str]]) -> list[dict]: + if str(i) in done_batches: + return done_batches[str(i)] + async with batch_sem: + batch_rows = await sweep_batch(index=i, total=len(batches), batch=b) + # every save rewrites the whole blob (runs list + all rows so far), a + # cost that grows with progress — so save on a time budget instead of + # per batch. A crash loses at most CHECKPOINT_SAVE_INTERVAL of work. + async with save_lock: + done_batches[str(i)] = batch_rows + now = time.monotonic() + if cp is not None and now - last_save["t"] >= CHECKPOINT_SAVE_INTERVAL: + await cp.save(json.dumps(state).encode()) + last_save["t"] = now + return batch_rows + + skipped = sum(1 for i in range(len(batches)) if str(i) in done_batches) + if skipped: + print(f"resuming: {skipped}/{len(batches)} batches restored from checkpoint", flush=True) + results = await asyncio.gather(*(run_batch(i, b) for i, b in enumerate(batches))) + rows: list[dict] = [r for batch_rows in results for r in batch_rows] + print(f"action rows: {len(rows)} (fanout-aggregated)", flush=True) + data = { + "rows": rows, + "users": user_names, + "org": get_init_config().org, + "window": [start or "", end or "now"], + "scope": f"{project or 'all projects'}/{domain or 'all domains'}", + } + # hand the rows back as an offloaded File: inline task outputs are capped + # (10MiB outputs.pb) and proto-Struct encoding is several times fatter + # than JSON, so a decent-size org overflows the cap long before the + # report itself is too big + import tempfile + + fd, tmp = tempfile.mkstemp(prefix="usage-rows-", suffix=".json") + os.close(fd) + await asyncio.to_thread(_write_json, tmp, data) + return await File.from_local(tmp) + + +@env.task(report=True) +async def usage_report( + start: str = "", + end: str = "", + project: str = "", + domain: str = "", + all_scopes: bool = False, + console_url: str = "", +) -> str: + """start/end: YYYY-MM-DD (default: last 12 full months + current month). + Default scope is the project/domain this task runs in; pass project/domain + to target another, or all_scopes=True to sweep the whole org.""" + if all_scopes: + project = domain = "" + elif not project: + project = os.environ.get("FLYTE_INTERNAL_EXECUTION_PROJECT", "") + domain = domain or os.environ.get("FLYTE_INTERNAL_EXECUTION_DOMAIN", "") + print(f"scope: {project or 'all projects'}/{domain or 'all domains'}", flush=True) + start_dt, end_dt = resolve_window(start, end) + # a stable checkpoint URI under THIS task's output prefix: both the normal + # sweep and a bigger-memory OOM re-run read/write the same blob, so + # completed batches survive the escalation (which a per-attempt checkpoint + # or @flyte.trace cannot — those die with the failed action) + tctx = flyte.ctx() + checkpoint_uri = "" + if tctx is not None and tctx.raw_data_path.path: + checkpoint_uri = tctx.raw_data_path.path.rstrip("/") + "/usage_sweep_checkpoint" + kwargs = { + "start": start_dt.date().isoformat() if start_dt else "", + "end": end_dt.date().isoformat() if end_dt else "", + "project": project, + "domain": domain, + "checkpoint_uri": checkpoint_uri, + } + try: + rows_file = await collect_actions(**kwargs) + except flyte.errors.OOMError as e: + # deterministic OOM won't be fixed by a same-size retry — re-run the + # sweep in a bigger container; the shared checkpoint_uri lets the new + # action resume from every batch the OOMed one completed + print(f"collect OOMed ({e.code}); retrying with {OOM_RETRY_RESOURCES}", flush=True) + rows_file = await collect_actions.override(resources=OOM_RETRY_RESOURCES)(**kwargs) + data = await asyncio.to_thread(_read_json, await rows_file.download()) + # for run links in the report; falls back to client-side origin detection + data["console"] = console_url.rstrip("/") + html = build_html(data) + await flyte.report.replace.aio(html) + await flyte.report.flush.aio() + n_actions = sum(r.get("cnt", 1) for r in data["rows"]) + return f"metered {n_actions} actions ({len(data['rows'])} rows) across org {data['org']}" + + +def _write_json(path: str, data: dict) -> None: + with open(path, "w", encoding="utf-8") as fh: + json.dump(data, fh, separators=(",", ":")) + + +def _read_json(path: str) -> dict: + with open(path, encoding="utf-8") as fh: + return json.load(fh) + + +def build_html(data: dict) -> str: + payload = json.dumps(data, separators=(",", ":")) + return _TEMPLATE.replace("__PAYLOAD__", payload.replace("", "<\\/")) + + +_TEMPLATE = r""" +