Skip to content

Commit bf33ef6

Browse files
committed
chore(server): sync discovery fixes from master
- merge the current Apache task and auth baseline - retain identity and auth cases in the combined fixture - compile both native readers with pinned dependencies
2 parents 91b00d2 + 87239c5 commit bf33ef6

38 files changed

Lines changed: 3511 additions & 407 deletions

‎.github/scripts/check-rerun.py‎

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import argparse
2020
import json
2121
import os
22+
import re
2223
import subprocess
2324
from urllib.parse import urlencode
2425

@@ -33,31 +34,69 @@ def api(path):
3334

3435
def decide(repository, run_id, expected_attempt, max_reruns, fetch=api):
3536
run = fetch(f"repos/{repository}/actions/runs/{run_id}")
36-
if run.get("path") not in {".github/workflows/server-memory-ci.yml", ".github/workflows/licence-checker.yml"}:
37-
return "skip", "only required workflows automatically retry"
37+
if run.get("path") != ".github/workflows/server-ci.yml":
38+
return "skip", "only Server CI required jobs automatically retry", None
3839
if (run.get("status") != "completed" or run.get("conclusion") != "failure"
3940
or run.get("run_attempt") != expected_attempt):
40-
return "skip", "source run changed or is no longer a completed failure"
41+
return "skip", "source run changed or is no longer a completed failure", None
4142
if expected_attempt > max_reruns:
42-
return "skip", "retry limit reached"
43+
return "skip", "retry limit reached", None
4344
sha = run.get("head_sha")
4445
if not sha:
45-
return "skip", "missing head SHA"
46+
return "skip", "missing head SHA", None
4647
if run.get("event") == "pull_request":
47-
return "skip", "PR automatic retry paused: CI artifacts cannot independently prove the tested code"
48+
return "skip", "PR automatic retry paused: CI artifacts cannot independently prove the tested code", None
4849
if run.get("event") == "push":
4950
if (run.get("repository", {}).get("full_name") != repository
5051
or run.get("head_repository", {}).get("full_name") != repository):
51-
return "skip", "push source repository differs"
52+
return "skip", "push source repository differs", None
5253
branch = run.get("head_branch")
5354
if not branch:
54-
return "skip", "missing push branch"
55+
return "skip", "missing push branch", None
5556
# Read from the workflow repository, never a similarly named fork branch.
5657
commits = fetch(f"repos/{repository}/commits?" + urlencode({"sha": branch, "per_page": 1}))
57-
if commits and commits[0].get("sha") == sha:
58-
return "rerun", "push branch head unchanged"
59-
return "skip", "push branch head moved"
60-
return "skip", "unsupported source event"
58+
if not commits or commits[0].get("sha") != sha:
59+
return "skip", "push branch head moved", None
60+
# A native job rerun includes its dependent jobs. Prefer a failed producer
61+
# over its failed reporter; never select the advisory branch of Server CI.
62+
targets = []
63+
page = 1
64+
while True:
65+
jobs = fetch(f"repos/{repository}/actions/runs/{run_id}/attempts/{expected_attempt}/jobs"
66+
f"?per_page=100&page={page}")["jobs"]
67+
for job in jobs:
68+
if job.get("status") != "completed" or job.get("conclusion") not in {"failure", "timed_out"}:
69+
continue
70+
name = job.get("name", "")
71+
if name == "check-license":
72+
priority = 0
73+
elif name == "memory-plan":
74+
priority = 1
75+
elif name == "memory-java-runtime / read-version":
76+
priority = 2
77+
elif re.fullmatch(r"build-server / server_memory \(Java [1-9][0-9]*(?:\.[0-9]+)*\)", name):
78+
priority = 3
79+
elif name == "Server memory tests":
80+
priority = 4
81+
else:
82+
continue
83+
# Only failed reporters retry themselves; a timed-out producer
84+
# must be retried before its dependent failure can recover.
85+
if priority == 4 and job.get("conclusion") != "failure":
86+
continue
87+
if job["run_id"] != run_id or job["head_sha"] != sha:
88+
return "skip", "job does not match source run", None
89+
job_id = job["id"]
90+
if not isinstance(job_id, int) or job_id <= 0:
91+
return "skip", "invalid job ID", None
92+
targets.append((priority, job_id))
93+
if len(jobs) < 100:
94+
break
95+
page += 1
96+
if targets:
97+
return "rerun", "failed required job and its dependencies", min(targets)[1]
98+
return "skip", "no failed required job in this attempt", None
99+
return "skip", "unsupported source event", None
61100

62101

63102
def main():
@@ -68,12 +107,12 @@ def main():
68107
parser.add_argument("--max-reruns", required=True, type=int)
69108
args = parser.parse_args()
70109
try:
71-
action, reason = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns)
110+
action, reason, job_id = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns)
72111
except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError):
73-
action, reason = "skip", "unable to verify current run and source metadata"
112+
action, reason, job_id = "skip", "unable to verify current run and source metadata", None
74113
print(f"{action}: {reason}")
75114
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
76-
output.write(f"action={action}\nreason={reason}\n")
115+
output.write(f"action={action}\nreason={reason}\njob_id={job_id or ''}\n")
77116

78117

79118
if __name__ == "__main__":

‎.github/scripts/test_check_rerun.py‎

Lines changed: 88 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import os
2222
import subprocess
2323
import tempfile
24+
import json
2425
from unittest.mock import patch
2526

2627
spec = importlib.util.spec_from_file_location("check_rerun", Path(__file__).with_name("check-rerun.py"))
@@ -32,31 +33,113 @@ class FreshnessTest(unittest.TestCase):
3233
def setUp(self):
3334
self.repository = "apache/hugegraph"
3435
self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1,
35-
"path": ".github/workflows/server-memory-ci.yml", "event": "push", "head_sha": "old", "head_branch": "release/test",
36+
"path": ".github/workflows/server-ci.yml", "event": "push", "head_sha": "old", "head_branch": "release/test",
3637
"repository": {"full_name": self.repository},
3738
"head_repository": {"full_name": self.repository}}
3839
self.current_head = "old"
3940
self.calls = []
41+
self.jobs = [self.job(117, "build-server / server_memory (Java 17)"),
42+
self.job(118, "Server memory tests")]
43+
44+
def job(self, job_id, name, conclusion="failure"):
45+
return {"id": job_id, "name": name, "status": "completed", "conclusion": conclusion,
46+
"run_id": 42, "head_sha": "old"}
4047

4148
def fetch(self, path):
4249
self.calls.append(path)
4350
if path == f"repos/{self.repository}/actions/runs/42":
4451
return self.run
4552
if path == f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1":
4653
return [{"sha": self.current_head}] if self.current_head else []
54+
if path.startswith(f"repos/{self.repository}/actions/runs/42/attempts/"):
55+
page = int(path.rsplit("page=", 1)[1])
56+
return {"jobs": self.jobs[(page - 1) * 100:page * 100]}
4757
raise AssertionError("Unexpected lookup: " + path)
4858

4959
def decide(self):
5060
return checker.decide(self.repository, 42, 1, 2, self.fetch)[0]
5161

52-
def test_only_required_workflows_retry(self):
53-
for path in [".github/workflows/server-ci.yml", ".github/workflows/codeql-analysis.yml",
54-
".github/workflows/server-compatibility-ci.yml"]:
62+
def test_only_server_required_jobs_retry(self):
63+
for path in [".github/workflows/server-memory-ci.yml", ".github/workflows/licence-checker.yml",
64+
".github/workflows/codeql-analysis.yml", ".github/workflows/server-compatibility-ci.yml"]:
5565
self.run["path"] = path
5666
self.assertEqual("skip", self.decide())
57-
self.run["path"] = ".github/workflows/licence-checker.yml"
67+
self.run["path"] = ".github/workflows/server-ci.yml"
5868
self.assertEqual("rerun", self.decide())
5969

70+
def test_failed_producer_is_selected_before_reporter(self):
71+
for name in ["check-license", "memory-plan", "memory-java-runtime / read-version",
72+
"build-server / server_memory (Java 17)"]:
73+
with self.subTest(name=name):
74+
self.jobs = [self.job(118, "Server memory tests"), self.job(117, name)]
75+
self.assertEqual(117, checker.decide(self.repository, 42, 1, 2, self.fetch)[2])
76+
self.jobs = [self.job(118, "Server memory tests")]
77+
self.assertEqual(118, checker.decide(self.repository, 42, 1, 2, self.fetch)[2])
78+
79+
def test_timed_out_producer_is_retried_only_inside_failed_run(self):
80+
self.jobs = [self.job(118, "Server memory tests"),
81+
self.job(117, "build-server / server_memory (Java 17)", "timed_out")]
82+
self.assertEqual(117, checker.decide(self.repository, 42, 1, 2, self.fetch)[2])
83+
self.run["conclusion"] = "timed_out"
84+
self.assertEqual("skip", self.decide())
85+
86+
def test_timed_out_reporter_alone_is_not_retried(self):
87+
self.jobs = [self.job(118, "Server memory tests", "timed_out")]
88+
self.assertEqual("skip", self.decide())
89+
90+
def test_advisory_success_and_non_failure_jobs_are_not_retried(self):
91+
for name in ["plan", "java-runtime / read-version", "server-rocksdb / server_rocksdb (Java 17)",
92+
"pd_store / struct", "affected-module-tests", "build-server / server_memory (Java 17) / extra"]:
93+
self.jobs = [self.job(118, name)]
94+
self.assertEqual("skip", self.decide())
95+
for conclusion in ["success", "cancelled", "skipped", None]:
96+
self.jobs = [self.job(118, "Server memory tests", conclusion)]
97+
self.assertEqual("skip", self.decide())
98+
99+
def test_job_selection_paginates_the_exact_attempt(self):
100+
self.jobs = [self.job(i + 1, "advisory") for i in range(100)] + [self.job(117, "memory-plan")]
101+
self.run["run_attempt"] = 2
102+
self.assertEqual(117, checker.decide(self.repository, 42, 2, 2, self.fetch)[2])
103+
self.assertIn(f"repos/{self.repository}/actions/runs/42/attempts/2/jobs?per_page=100&page=2", self.calls)
104+
self.assertFalse(any("/attempts/1/" in path for path in self.calls))
105+
106+
def test_job_identity_and_metadata_must_match(self):
107+
for key, value in [("run_id", 99), ("head_sha", "different"), ("id", "117"), ("id", -1)]:
108+
with self.subTest(key=key):
109+
self.jobs = [dict(self.job(117, "memory-plan"), **{key: value})]
110+
self.assertEqual("skip", self.decide())
111+
112+
def test_workflow_posts_native_rerun_for_checker_selected_job(self):
113+
workflow = Path(__file__).resolve().parents[1] / "workflows/rerun-ci.yml"
114+
source = workflow.read_text()
115+
self.assertIn(' - "HugeGraph-Server CI"', source)
116+
self.assertIn("github.event.workflow_run.event == 'push'", source)
117+
self.assertIn("ref: ${{ github.sha }}", source)
118+
command = source.split(" - name: Rerun the required job and its dependencies", 1)[1]
119+
command = command.split(" run: |\n", 1)[1].strip()
120+
with tempfile.TemporaryDirectory() as directory:
121+
root = Path(directory)
122+
output = root / "output"
123+
decide = checker.decide
124+
with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch(
125+
"sys.argv", ["check-rerun.py", "--repository", self.repository,
126+
"--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object(
127+
checker, "decide", side_effect=lambda *args: decide(*args, fetch=self.fetch)):
128+
checker.main()
129+
selected = dict(line.split("=", 1) for line in output.read_text().splitlines())
130+
self.assertEqual("rerun", selected["action"])
131+
command = command.replace("${{ steps.freshness.outputs.job_id }}", selected["job_id"])
132+
gh = root / "gh"
133+
gh.write_text('#!/usr/bin/env python3\nimport json, os, sys\n'
134+
'open(os.environ["ARGS_PATH"], "w").write(json.dumps(sys.argv[1:]))\n')
135+
gh.chmod(0o700)
136+
args_path = root / "args.json"
137+
subprocess.run(["bash", "-eu", "-c", command], check=True,
138+
env={**os.environ, "GH_REPO": self.repository, "ARGS_PATH": str(args_path),
139+
"PATH": str(root) + os.pathsep + os.environ["PATH"]})
140+
self.assertEqual(["api", "--method", "POST", f"repos/{self.repository}/actions/jobs/117/rerun"],
141+
json.loads(args_path.read_text()))
142+
60143
def test_api_failure_fails_closed(self):
61144
with tempfile.TemporaryDirectory() as directory:
62145
output = Path(directory) / "output"

‎.github/scripts/test_ci_policy.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from pathlib import Path
2222
import subprocess
2323
import tempfile
24+
import textwrap
2425
import unittest
2526
from unittest.mock import patch
2627

@@ -197,6 +198,45 @@ def plan(self):
197198
"testedMergeSHA": "merge", "expected": ["server_memory", "server_rocksdb", "pd_store", "cluster"],
198199
"selected": ["server", "pd", "store", "hstore", "cluster"]}
199200

201+
def memory_workflow_results(self, planner="success", runtime="success", build="success", selected=True):
202+
workflow = Path(__file__).resolve().parents[1] / "workflows/server-ci.yml"
203+
reporter = workflow.read_text().split("\n server-memory-required:\n", 1)[1]
204+
code = reporter.split("python3 - <<'PYGATE'\n", 1)[1].split("\n PYGATE", 1)[0]
205+
# Start with the actual needs keys, without generic gate aliases.
206+
needs = {"memory-plan": {"result": planner, "outputs": {"server": str(selected).lower()}},
207+
"memory-java-runtime": {"result": runtime}, "build-server": {"result": build}}
208+
with tempfile.TemporaryDirectory() as directory:
209+
outcome = subprocess.run(["python3", "-c", textwrap.dedent(code)], cwd=directory,
210+
env={**os.environ, "JOB_RESULTS": json.dumps(needs)},
211+
capture_output=True, text=True)
212+
if outcome.returncode:
213+
return outcome, None
214+
return outcome, json.loads((Path(directory) / "results.json").read_text())
215+
216+
def test_memory_workflow_results_feed_real_gate(self):
217+
for state in ["success", "failure", "cancelled", "skipped", None]:
218+
with self.subTest(build=state):
219+
outcome, results = self.memory_workflow_results(build=state)
220+
self.assertEqual(0, outcome.returncode, outcome.stderr)
221+
if state == "success":
222+
self.assertEqual(["server_memory"], policy.gate(self.plan(), results, mode="memory")["executed"])
223+
else:
224+
with self.assertRaisesRegex(policy.GateFailure, "server_memory"):
225+
policy.gate(self.plan(), results, mode="memory")
226+
outcome, results = self.memory_workflow_results(runtime="skipped", build="skipped", selected=False)
227+
self.assertEqual(0, outcome.returncode, outcome.stderr)
228+
docs = dict(self.plan(), expected=[], selected=[])
229+
self.assertEqual([], policy.gate(docs, results, mode="memory")["executed"])
230+
for state in ["failure", "cancelled", "skipped"]:
231+
with self.subTest(planner=state):
232+
outcome, results = self.memory_workflow_results(planner=state)
233+
self.assertNotEqual(0, outcome.returncode)
234+
self.assertIn("CI planning failed", outcome.stderr)
235+
with self.subTest(runtime=state):
236+
outcome, results = self.memory_workflow_results(runtime=state)
237+
self.assertNotEqual(0, outcome.returncode)
238+
self.assertIn("Selected Java runtime preparation failed", outcome.stderr)
239+
200240
def test_memory_gate_ignores_advisory_cancellation(self):
201241
plan = self.plan()
202242
results = {"plan": {"result": "success"}, "server_memory": {"result": "success"},

‎.github/workflows/licence-checker.yml‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎.github/workflows/rerun-ci.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ name: "Rerun CI"
33
on:
44
workflow_run:
55
workflows:
6-
- "Server Memory CI"
7-
- "License Checker"
6+
- "HugeGraph-Server CI"
87
types:
98
- completed
109

@@ -108,10 +107,10 @@ jobs:
108107
--repository "$GH_REPO" --run-id "$RUN_ID" \
109108
--run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS"
110109
111-
- name: Rerun failed jobs
110+
- name: Rerun the required job and its dependencies
112111
if: steps.freshness.outputs.action == 'rerun'
113112
env:
114113
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
115114
GH_REPO: ${{ github.repository }}
116115
run: |
117-
gh run rerun ${{ github.event.workflow_run.id }} --failed
116+
gh api --method POST "repos/$GH_REPO/actions/jobs/${{ steps.freshness.outputs.job_id }}/rerun"

0 commit comments

Comments
 (0)