Skip to content

Commit 44f0e8f

Browse files
wchilders-nvidiawchilders-nvidia
committed
Initial implementation of AcknowlEDG Forge for integrating software forges with AcknowlEDG []
1 parent 9d0b0a8 commit 44f0e8f

96 files changed

Lines changed: 15297 additions & 59 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/acknowledg-deploy.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
# Host, paths, and credentials come from secrets in the `acknowledg`
1717
# environment (see dev_tools/services/acknowledg/README.md). The SSH setup
1818
# writes them into a `Host acknowledg` alias so they never appear on a command
19-
# line. The public API URL is the ACKNOWLEDG_API_HOST repository variable.
19+
# line. The public API URL is the ACKNOWLEDG_API_HOST repository variable,
20+
# shared with acknowledg-forge-deploy.yml.
2021

2122
name: AcknowlEDG Deploy
2223

@@ -27,6 +28,8 @@ on:
2728
paths:
2829
- dev_tools/services/acknowledg/**
2930
- dev_tools/pylibs/edgacknowledg/**
31+
# acknowledg-forge is a separate service.
32+
- '!dev_tools/pylibs/edgacknowledg/forge/**'
3033
- .github/workflows/acknowledg-deploy.yml
3134
workflow_dispatch:
3235

‎.github/workflows/acknowledg-forge-deploy.yml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
2+
# Exceptions.
3+
# See https://edgcpp.org/LICENSE.txt for license information.
4+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5+
#
6+
# Publish the acknowledg-forge container image and deploy it:
7+
#
8+
# test the acknowledg-forge typecheck and tests (acknowledg-forge.yml)
9+
# publish build the image (build-edg-env acknowledg-forge) and push it to
10+
# ghcr.io/edgcpp/acknowledg-forge:latest (deployments use the
11+
# pushed digest)
12+
# deploy over SSH, run the host's deploy helper (an administrator-
13+
# installed copy of
14+
# dev_tools/services/acknowledg-forge/deploy-remote.py, the only
15+
# thing the deploy user may run as root) to point the service at
16+
# the pushed digest, set its AcknowlEDG API and UI URLs, and
17+
# restart it (only if any of those changed)
18+
#
19+
# The image is public and holds no credentials or configuration: the publish
20+
# job has no access to the deployment's secrets. Everything else about the
21+
# service (its unit, tokens, API key) is configured on the host.
22+
#
23+
# Host, user, and SSH credentials come from secrets in the
24+
# `acknowledg-forge` environment; the public AcknowlEDG URLs come from
25+
# repository variables shared with acknowledg-deploy.yml (see
26+
# dev_tools/services/acknowledg-forge/README.md).
27+
28+
name: acknowledg-forge Deploy
29+
30+
on:
31+
push:
32+
branches:
33+
- main
34+
paths:
35+
# The image freezes all of dev_tools/bin and dev_tools/pylibs.
36+
- dev_tools/bin/**
37+
- dev_tools/pylibs/**
38+
- dev_tools/services/acknowledg-forge/**
39+
- dev_tools/docker/acknowledg-forge/**
40+
- .github/workflows/acknowledg-forge.yml
41+
- .github/workflows/acknowledg-forge-deploy.yml
42+
workflow_dispatch:
43+
44+
concurrency:
45+
group: acknowledg-forge-deploy
46+
cancel-in-progress: false
47+
48+
permissions:
49+
contents: read
50+
51+
jobs:
52+
test:
53+
name: Test
54+
if: github.ref == 'refs/heads/main'
55+
uses: ./.github/workflows/acknowledg-forge.yml
56+
57+
publish:
58+
name: Publish image
59+
needs: test
60+
runs-on: ubuntu-latest
61+
timeout-minutes: 30
62+
permissions:
63+
contents: read
64+
packages: write
65+
outputs:
66+
image: ${{ steps.push.outputs.image }}
67+
steps:
68+
- name: Check out repository
69+
uses: actions/checkout@v7
70+
71+
- name: Set up uv
72+
uses: astral-sh/setup-uv@v10.3.0
73+
74+
- name: Build image
75+
env:
76+
PYTHONPATH: ${{ github.workspace }}/dev_tools/pylibs
77+
run: dev_tools/bin/build-edg-env --quiet acknowledg-forge
78+
79+
- name: Push image
80+
id: push
81+
env:
82+
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
83+
run: |
84+
image=ghcr.io/edgcpp/acknowledg-forge
85+
printf '%s' "${REGISTRY_TOKEN}" \
86+
| docker login ghcr.io --username "${GITHUB_ACTOR}" \
87+
--password-stdin
88+
docker push --quiet "${image}:latest"
89+
docker logout ghcr.io
90+
digest="$(docker inspect \
91+
--format '{{range .RepoDigests}}{{println .}}{{end}}' \
92+
"${image}:latest" | grep "^${image}@sha256:" | head -n 1)"
93+
echo "Pushed ${digest}"
94+
echo "image=${digest}" >> "${GITHUB_OUTPUT}"
95+
96+
deploy:
97+
name: Deploy
98+
needs: publish
99+
runs-on: ubuntu-latest
100+
timeout-minutes: 15
101+
environment: acknowledg-forge
102+
steps:
103+
- name: Configure SSH
104+
env:
105+
SSH_HOST: ${{ secrets.ACKNOWLEDG_FORGE_SSH_HOST }}
106+
SSH_USER: ${{ secrets.ACKNOWLEDG_FORGE_SSH_USER }}
107+
SSH_PORT: ${{ secrets.ACKNOWLEDG_FORGE_SSH_PORT || '22' }}
108+
SSH_PRIVATE_KEY: ${{ secrets.ACKNOWLEDG_FORGE_SSH_PRIVATE_KEY }}
109+
SSH_KNOWN_HOSTS: ${{ secrets.ACKNOWLEDG_FORGE_SSH_KNOWN_HOSTS }}
110+
run: |
111+
install -d -m 700 ~/.ssh
112+
install -m 600 /dev/null ~/.ssh/deploy_key
113+
printf '%s\n' "${SSH_PRIVATE_KEY}" > ~/.ssh/deploy_key
114+
printf '%s\n' "${SSH_KNOWN_HOSTS}" > ~/.ssh/known_hosts
115+
cat > ~/.ssh/config <<EOF
116+
Host acknowledg-forge
117+
HostName ${SSH_HOST}
118+
User ${SSH_USER}
119+
Port ${SSH_PORT}
120+
IdentityFile ~/.ssh/deploy_key
121+
IdentitiesOnly yes
122+
StrictHostKeyChecking yes
123+
BatchMode yes
124+
EOF
125+
chmod 600 ~/.ssh/config ~/.ssh/known_hosts
126+
127+
- name: Update and restart the service
128+
env:
129+
SERVICE: ${{ secrets.ACKNOWLEDG_FORGE_SERVICE || 'acknowledg-forge-github' }}
130+
IMAGE: ${{ needs.publish.outputs.image }}
131+
API_URL: ${{ vars.ACKNOWLEDG_API_HOST }}
132+
UI_URL: ${{ vars.ACKNOWLEDG_UI_HOST }}
133+
run: |
134+
# The arguments are deliberately expanded (and quoted) here.
135+
# shellcheck disable=SC2029
136+
ssh acknowledg-forge "sudo -n /usr/local/libexec/acknowledg-forge-deploy \
137+
$(printf '%q ' "${SERVICE}" "${IMAGE}" "${API_URL}" \
138+
"${UI_URL}")"

‎.github/workflows/acknowledg-forge.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
2+
# Exceptions.
3+
# See https://edgcpp.org/LICENSE.txt for license information.
4+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5+
#
6+
# Typecheck and test acknowledg-forge: the shared edgacknowledg.forge
7+
# package, the GitHub and GitLab services, and their integration with the
8+
# AcknowlEDG client library and server.
9+
10+
name: acknowledg-forge
11+
12+
on:
13+
push:
14+
branches:
15+
- main
16+
paths:
17+
- dev_tools/pylibs/edgacknowledg/**
18+
- dev_tools/pylibs/edgtest/**
19+
- dev_tools/pylibs/edgutil/**
20+
- dev_tools/services/acknowledg/server/**
21+
- dev_tools/services/acknowledg-forge/**
22+
- dev_tools/docker/acknowledg-forge/**
23+
- .github/workflows/acknowledg-forge.yml
24+
pull_request:
25+
paths:
26+
- dev_tools/pylibs/edgacknowledg/**
27+
- dev_tools/pylibs/edgtest/**
28+
- dev_tools/pylibs/edgutil/**
29+
- dev_tools/services/acknowledg/server/**
30+
- dev_tools/services/acknowledg-forge/**
31+
- dev_tools/docker/acknowledg-forge/**
32+
- .github/workflows/acknowledg-forge.yml
33+
# Also run by acknowledg-forge-deploy before publishing an image.
34+
workflow_call:
35+
36+
permissions:
37+
contents: read
38+
39+
jobs:
40+
test:
41+
name: Typecheck and test
42+
runs-on: ubuntu-latest
43+
timeout-minutes: 20
44+
steps:
45+
- name: Check out repository
46+
uses: actions/checkout@v7
47+
48+
- name: Set up uv
49+
uses: astral-sh/setup-uv@v10.3.0
50+
51+
- name: Typecheck
52+
run: dev_tools/services/acknowledg-forge/typecheck.sh
53+
54+
- name: Test
55+
run: dev_tools/services/acknowledg-forge/tests/run-tests.sh -v
56+
57+
containers:
58+
# The deployment itself (compose.yaml): what each container sees and
59+
# can reach, on the runner's stock Docker and AppArmor (nothing to set
60+
# up on the host, which is the point), on the Ubuntu release deployments
61+
# run on.
62+
name: Containers
63+
runs-on: ubuntu-26.04
64+
timeout-minutes: 30
65+
steps:
66+
- name: Check out repository
67+
uses: actions/checkout@v7
68+
69+
- name: Set up uv
70+
uses: astral-sh/setup-uv@v10.3.0
71+
72+
- name: Build image
73+
env:
74+
PYTHONPATH: ${{ github.workspace }}/dev_tools/pylibs
75+
run: dev_tools/bin/build-edg-env --quiet acknowledg-forge
76+
77+
- name: Test
78+
env:
79+
ACKNOWLEDG_FORGE_TEST_IMAGE: ghcr.io/edgcpp/acknowledg-forge:latest
80+
run: |
81+
dev_tools/services/acknowledg-forge/tests/run-tests.sh -v -rs \
82+
test_containers.py

‎dev_tools/bin/edg-rematerialize-test-run‎

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ def collect_affected_tests(test_home: Path,
122122

123123
tests_dir = test_home / 'tests'
124124
expectations_dir = test_home / 'expectations'
125+
# The test files themselves and the default recordings are what the default
126+
# configuration uses.
127+
default_config = edgtest.runtest.get_default_config_name()
125128

126129
for diff_file in diff_files:
127130
file_path = test_home.parent.joinpath(PurePosixPath(diff_file.file_name))
@@ -130,10 +133,12 @@ def collect_affected_tests(test_home: Path,
130133
# A file in a test suite.
131134
orig_file_path = file_path
132135

133-
file_extensions = ''.join(file_path.suffixes)
136+
# The extensions are listed without their leading '.'.
137+
file_extensions = ''.join(file_path.suffixes)[1:]
134138
if file_extensions in edgtest.DEFAULT_TEST_EXTENSIONS:
135139
# A test file itself.
136140
result.tests.add(file_path)
141+
result.configs.add(default_config)
137142
continue
138143

139144
test_file_found = False
@@ -154,8 +159,11 @@ def collect_affected_tests(test_home: Path,
154159
)
155160
if candidate_test_file.exists():
156161
result.tests.add(candidate_test_file)
157-
if orig_file_path.stem != 'default':
158-
result.configs.add(orig_file_path.stem)
162+
# Recordings are named <config>.<case>.<command>.txt.
163+
recording_config = orig_file_path.name.split('.')[0]
164+
if recording_config == 'default':
165+
recording_config = default_config
166+
result.configs.add(recording_config)
159167
break
160168
test_file_found = True
161169
else:
@@ -195,12 +203,16 @@ async def patch_file(output_dir: Path,
195203
input_dir: Path,
196204
relative_path: PurePosixPath,
197205
diff_file_path: Path) -> bool:
206+
output_path = output_dir / relative_path
207+
# A new file (e.g. a new test's recording) has no copied directory to be
208+
# written into.
209+
output_path.parent.mkdir(parents = True, exist_ok = True)
198210
patch_proc = await asyncio.create_subprocess_exec(
199211
str(TOOL_PATH_PATCH),
200-
str(input_dir.joinpath(relative_path)),
212+
str(input_dir / relative_path),
201213
str(diff_file_path),
202214
'-o',
203-
str(output_dir.joinpath(relative_path)),
215+
str(output_path),
204216
stdout = asyncio.subprocess.DEVNULL
205217
)
206218
await patch_proc.wait()
@@ -236,6 +248,9 @@ async def materialize_tests(loop: asyncio.AbstractEventLoop,
236248

237249
# First create a copy of all relevant tests.
238250
for test in test_change_state.tests:
251+
if not test.exists():
252+
# A new test; the patch creates all of its files.
253+
continue
239254
relative_dir = test.parent.relative_to(tests_dir)
240255
edgtest.copy_raw_test_files(test, output_dir.joinpath(relative_dir))
241256

@@ -375,7 +390,7 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
375390

376391
command_metadata = []
377392
matches_metadata = None
378-
any_mismatch = False
393+
comparisons: List[edgtest.exec.OutputComparison] = []
379394

380395
for command_number in range(1, num_commands + 1):
381396
cmd_dir = case_dir / str(command_number)
@@ -427,8 +442,7 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
427442
'change.diff'
428443
)
429444

430-
if output_cmp_result != edgtest.exec.OutputComparison.EQUAL:
431-
any_mismatch = True
445+
comparisons.append(output_cmp_result)
432446

433447
command_metadata.append({
434448
'command': command_str,
@@ -471,7 +485,14 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
471485
'matches',
472486
'change.diff'
473487
)
474-
any_mismatch = True
488+
489+
# As edgy compares a case's matches (process_matches_mismatch).
490+
if old_matches is None:
491+
comparisons.append(edgtest.exec.OutputComparison.PREV_MISSING)
492+
elif matches_diff_hash is None:
493+
comparisons.append(edgtest.exec.OutputComparison.EQUAL)
494+
else:
495+
comparisons.append(edgtest.exec.OutputComparison.MISMATCH)
475496

476497
matches_metadata = {'diff_hash': matches_diff_hash}
477498

@@ -482,9 +503,13 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
482503
old_status = get_status_for_test_key(old_expectations, key_str)
483504
new_status = get_status_for_test_key(new_expectations, key_str)
484505

485-
# Add the mismatch string if needed.
486-
if any_mismatch:
506+
# Add the output comparison string as edgy does (e.g. a new test has no
507+
# previous output).
508+
case_comparison = edgtest.exec.aggregate_output_comparison(comparisons)
509+
if case_comparison == edgtest.exec.OutputComparison.MISMATCH:
487510
new_status = f"{new_status}--OUTPUT MISMATCH"
511+
elif case_comparison == edgtest.exec.OutputComparison.PREV_MISSING:
512+
new_status = f"{new_status}--NO PREVIOUS OUTPUT"
488513

489514
case_entry = {
490515
'status': {
@@ -625,6 +650,7 @@ async def simulate_config_testing(
625650
new_parent_dir
626651
)
627652

653+
tests = sorted(tests)
628654
for test_path in tests:
629655
rel_test_path_str = str(
630656
test_path.relative_to(test_home / 'tests')
@@ -647,6 +673,19 @@ async def simulate_config_testing(
647673
for status_line in test_status_lines:
648674
file_handle.write(status_line)
649675

676+
# As edgy does, list the tests whose status changed (by their path relative
677+
# to the tests directory, which is where their .rt.tar files are in the
678+
# configuration's run directory) so edgy-review and others can find them.
679+
changed_tests = [
680+
test_path.relative_to(test_home / 'tests').as_posix()
681+
for test_path, test_status_lines in zip(tests, tests_status_lines)
682+
if len(test_status_lines) > 0
683+
]
684+
if len(changed_tests) > 0:
685+
with open(config_run_dir / 'changes.elist', 'w') as file_handle:
686+
for changed_test in changed_tests:
687+
file_handle.write(f"{changed_test}\n")
688+
650689
async def simulate_testing(loop: asyncio.AbstractEventLoop,
651690
test_home: Path, new_run_dir: Path,
652691
patched_tests_dir: Path,

0 commit comments

Comments
 (0)