Skip to content

Commit 89dcd39

Browse files
committed
fix(ci): address sdk proto sync review feedback
Signed-off-by: Ignas Baranauskas <ibaranau@redhat.com>
1 parent 9da602e commit 89dcd39

8 files changed

Lines changed: 436 additions & 363 deletions

File tree

‎.github/workflows/sdk-proto-check.yml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,29 @@ jobs:
2626
pr_metadata:
2727
name: Resolve PR metadata
2828
runs-on: ubuntu-latest
29+
timeout-minutes: 5
2930
permissions:
3031
contents: read
3132
pull-requests: read
3233
outputs:
3334
should_run: ${{ steps.gate.outputs.should_run }}
35+
matrix: ${{ steps.config.outputs.matrix }}
3436
steps:
3537
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
3638

3739
- id: gate
3840
uses: ./.github/actions/pr-gate
3941

42+
- id: config
43+
name: Load SDK configuration
44+
run: echo "matrix=$(jq -c '.include' tasks/sdk-sync-config.json)" >> "$GITHUB_OUTPUT"
45+
4046
sdk_proto_drift:
4147
name: Proto Drift (${{ matrix.sdk.name }})
4248
needs: pr_metadata
4349
if: needs.pr_metadata.outputs.should_run == 'true'
4450
runs-on: linux-amd64-cpu8
51+
timeout-minutes: 15
4552
container:
4653
image: ghcr.io/nvidia/openshell/ci:latest
4754
credentials:
@@ -50,11 +57,7 @@ jobs:
5057
strategy:
5158
fail-fast: false
5259
matrix:
53-
sdk:
54-
- name: go
55-
drift_task: "go:proto:drift"
56-
- name: typescript
57-
drift_task: "sdk:ts:proto:drift"
60+
sdk: ${{ fromJSON(needs.pr_metadata.outputs.matrix) }}
5861
steps:
5962
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
6063

@@ -66,12 +69,13 @@ jobs:
6669
run: |
6770
REPORT=$(mise run ${{ matrix.sdk.drift_task }} 2>"$RUNNER_TEMP/drift_stderr.log") || true
6871
69-
if echo "$REPORT" | jq -e '.synced' >/dev/null 2>&1; then
72+
if echo "$REPORT" | jq -e 'has("synced") and (.synced | type == "boolean")' >/dev/null 2>&1; then
7073
SYNCED=$(echo "$REPORT" | jq -r '.synced')
74+
DELIMITER="REPORT_EOF_$(openssl rand -hex 16)"
7175
{
72-
echo "report<<REPORT_EOF"
76+
echo "report<<$DELIMITER"
7377
echo "$REPORT"
74-
echo "REPORT_EOF"
78+
echo "$DELIMITER"
7579
} >> "$GITHUB_OUTPUT"
7680
echo "synced=$SYNCED" >> "$GITHUB_OUTPUT"
7781
else

‎.github/workflows/sdk-sync-dashboard.yml‎

Lines changed: 67 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -22,98 +22,74 @@ concurrency:
2222
cancel-in-progress: true
2323

2424
jobs:
25+
load_config:
26+
name: Load SDK configuration
27+
runs-on: ubuntu-latest
28+
timeout-minutes: 5
29+
outputs:
30+
matrix: ${{ steps.config.outputs.matrix }}
31+
steps:
32+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
33+
- id: config
34+
run: echo "matrix=$(jq -c '.include' tasks/sdk-sync-config.json)" >> "$GITHUB_OUTPUT"
35+
2536
sdk_sync_check:
26-
name: Sync Check
37+
name: Sync Check (${{ matrix.sdk.name }})
38+
needs: load_config
2739
runs-on: linux-amd64-cpu8
40+
timeout-minutes: 30
2841
container:
2942
image: ghcr.io/nvidia/openshell/ci:latest
3043
credentials:
3144
username: ${{ github.actor }}
3245
password: ${{ secrets.GITHUB_TOKEN }}
33-
outputs:
34-
go_drift_report: ${{ steps.go_drift.outputs.report }}
35-
go_has_drift: ${{ steps.go_drift.outputs.has_drift }}
36-
go_build_report: ${{ steps.go_build.outputs.report }}
37-
go_build_failed: ${{ steps.go_build.outputs.build_failed || 'false' }}
38-
ts_drift_report: ${{ steps.ts_drift.outputs.report }}
39-
ts_has_drift: ${{ steps.ts_drift.outputs.has_drift }}
40-
ts_build_report: ${{ steps.ts_build.outputs.report }}
41-
ts_build_failed: ${{ steps.ts_build.outputs.build_failed || 'false' }}
46+
strategy:
47+
fail-fast: false
48+
matrix:
49+
sdk: ${{ fromJSON(needs.load_config.outputs.matrix) }}
4250
steps:
4351
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
44-
4552
- name: Install tools
4653
run: mise install --locked
47-
48-
- name: Check Go proto drift
49-
id: go_drift
50-
run: |
51-
REPORT=$(mise run go:proto:drift 2>"$RUNNER_TEMP/go_drift_stderr.log") || true
52-
if echo "$REPORT" | jq -e '.synced' >/dev/null 2>&1; then
53-
SYNCED=$(echo "$REPORT" | jq -r '.synced')
54-
{
55-
echo "report<<REPORT_EOF"
56-
echo "$REPORT"
57-
echo "REPORT_EOF"
58-
} >> "$GITHUB_OUTPUT"
59-
[ "$SYNCED" = "true" ] && echo "has_drift=false" >> "$GITHUB_OUTPUT" || echo "has_drift=true" >> "$GITHUB_OUTPUT"
60-
else
61-
echo "::error::Go proto drift check failed"
62-
echo "stderr: $(cat "$RUNNER_TEMP/go_drift_stderr.log")"
63-
echo "report={}" >> "$GITHUB_OUTPUT"
64-
echo "has_drift=error" >> "$GITHUB_OUTPUT"
65-
fi
66-
67-
- name: Go build check
68-
id: go_build
69-
if: steps.go_drift.outputs.has_drift == 'true'
54+
- name: Check proto drift and build
55+
env:
56+
SDK_NAME: ${{ matrix.sdk.name }}
57+
DRIFT_TASK: ${{ matrix.sdk.drift_task }}
58+
BUILD_CHECK_TASK: ${{ matrix.sdk.build_check_task }}
7059
run: |
71-
REPORT=$(mise run go:proto:build-check 2>"$RUNNER_TEMP/go_build_stderr.log") && BUILD_OK=true || BUILD_OK=false
72-
if echo "$REPORT" | jq -e '.sdk' >/dev/null 2>&1; then
73-
{ echo "report<<REPORT_EOF"; echo "$REPORT"; echo "REPORT_EOF"; } >> "$GITHUB_OUTPUT"
74-
else
75-
echo "stderr: $(cat "$RUNNER_TEMP/go_build_stderr.log")"
76-
echo "report={}" >> "$GITHUB_OUTPUT"
60+
mkdir -p report
61+
mise run "$DRIFT_TASK" > report/drift.json 2> report/drift.stderr || true
62+
if ! jq -e 'has("synced") and (.synced | type == "boolean")' report/drift.json >/dev/null 2>&1; then
63+
jq -n --arg sdk "$SDK_NAME" '{sdk:$sdk, has_drift:"error", build_failed:"false"}' > report/status.json
64+
exit 0
7765
fi
78-
[ "$BUILD_OK" = "true" ] && echo "build_failed=false" >> "$GITHUB_OUTPUT" || echo "build_failed=true" >> "$GITHUB_OUTPUT"
79-
80-
- name: Check TypeScript proto drift
81-
id: ts_drift
82-
run: |
83-
REPORT=$(mise run sdk:ts:proto:drift 2>"$RUNNER_TEMP/ts_drift_stderr.log") || true
84-
if echo "$REPORT" | jq -e '.synced' >/dev/null 2>&1; then
85-
SYNCED=$(echo "$REPORT" | jq -r '.synced')
86-
{
87-
echo "report<<REPORT_EOF"
88-
echo "$REPORT"
89-
echo "REPORT_EOF"
90-
} >> "$GITHUB_OUTPUT"
91-
[ "$SYNCED" = "true" ] && echo "has_drift=false" >> "$GITHUB_OUTPUT" || echo "has_drift=true" >> "$GITHUB_OUTPUT"
92-
else
93-
echo "::error::TypeScript proto drift check failed"
94-
echo "stderr: $(cat "$RUNNER_TEMP/ts_drift_stderr.log")"
95-
echo "report={}" >> "$GITHUB_OUTPUT"
96-
echo "has_drift=error" >> "$GITHUB_OUTPUT"
66+
SYNCED=$(jq -r '.synced' report/drift.json)
67+
if [ "$SYNCED" = "true" ]; then
68+
jq -n --arg sdk "$SDK_NAME" '{sdk:$sdk, has_drift:"false", build_failed:"false"}' > report/status.json
69+
exit 0
9770
fi
98-
99-
- name: TypeScript build check
100-
id: ts_build
101-
if: steps.ts_drift.outputs.has_drift == 'true'
102-
run: |
103-
REPORT=$(mise run sdk:ts:proto:build-check 2>"$RUNNER_TEMP/ts_build_stderr.log") && BUILD_OK=true || BUILD_OK=false
104-
if echo "$REPORT" | jq -e '.sdk' >/dev/null 2>&1; then
105-
{ echo "report<<REPORT_EOF"; echo "$REPORT"; echo "REPORT_EOF"; } >> "$GITHUB_OUTPUT"
71+
if mise run "$BUILD_CHECK_TASK" > report/build.json 2> report/build.stderr; then
72+
BUILD_FAILED=false
10673
else
107-
echo "stderr: $(cat "$RUNNER_TEMP/ts_build_stderr.log")"
108-
echo "report={}" >> "$GITHUB_OUTPUT"
74+
BUILD_FAILED=true
10975
fi
110-
[ "$BUILD_OK" = "true" ] && echo "build_failed=false" >> "$GITHUB_OUTPUT" || echo "build_failed=true" >> "$GITHUB_OUTPUT"
76+
jq -n --arg sdk "$SDK_NAME" --arg build_failed "$BUILD_FAILED" \
77+
'{sdk:$sdk, has_drift:"true", build_failed:$build_failed}' > report/status.json
78+
- name: Upload SDK report
79+
if: always()
80+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
81+
with:
82+
name: sdk-sync-${{ matrix.sdk.name }}
83+
path: report/
84+
if-no-files-found: error
85+
retention-days: 1
11186

11287
issue_management:
11388
name: Manage Drift Issue (${{ matrix.sdk.name }})
114-
needs: sdk_sync_check
115-
if: always() && needs.sdk_sync_check.result == 'success'
89+
needs: [load_config, sdk_sync_check]
90+
if: always() && needs.load_config.result == 'success' && needs.sdk_sync_check.result == 'success'
11691
runs-on: linux-amd64-cpu8
92+
timeout-minutes: 5
11793
container:
11894
image: ghcr.io/nvidia/openshell/ci:latest
11995
credentials:
@@ -122,43 +98,36 @@ jobs:
12298
strategy:
12399
fail-fast: false
124100
matrix:
125-
sdk:
126-
- name: go
127-
label: "sdk:go:sync"
128-
has_drift: ${{ needs.sdk_sync_check.outputs.go_has_drift }}
129-
build_failed: ${{ needs.sdk_sync_check.outputs.go_build_failed }}
130-
drift_report: ${{ needs.sdk_sync_check.outputs.go_drift_report }}
131-
build_report: ${{ needs.sdk_sync_check.outputs.go_build_report }}
132-
- name: typescript
133-
label: "sdk:typescript:sync"
134-
has_drift: ${{ needs.sdk_sync_check.outputs.ts_has_drift }}
135-
build_failed: ${{ needs.sdk_sync_check.outputs.ts_build_failed }}
136-
drift_report: ${{ needs.sdk_sync_check.outputs.ts_drift_report }}
137-
build_report: ${{ needs.sdk_sync_check.outputs.ts_build_report }}
101+
sdk: ${{ fromJSON(needs.load_config.outputs.matrix) }}
138102
steps:
103+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
104+
with:
105+
name: sdk-sync-${{ matrix.sdk.name }}
106+
path: report
107+
- name: Read SDK status
108+
id: status
109+
run: |
110+
echo "has_drift=$(jq -r '.has_drift' report/status.json)" >> "$GITHUB_OUTPUT"
111+
echo "build_failed=$(jq -r '.build_failed' report/status.json)" >> "$GITHUB_OUTPUT"
139112
- name: Warn on drift detection error
140-
if: matrix.sdk.has_drift == 'error'
113+
if: steps.status.outputs.has_drift == 'error'
141114
run: |
142-
echo "::error::Drift detection failed for ${{ matrix.sdk.name }} SDK — check the sdk_sync_check job logs"
115+
echo "::error::Drift detection failed for ${{ matrix.sdk.name }} SDK"
116+
cat report/drift.stderr
143117
exit 1
144-
145-
- if: matrix.sdk.has_drift == 'true' && matrix.sdk.build_failed == 'true'
118+
- if: steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'true'
146119
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
147-
148120
- name: Install tools
149-
if: matrix.sdk.has_drift == 'true' && matrix.sdk.build_failed == 'true'
121+
if: steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'true'
150122
run: mise install --locked
151-
152123
- name: Create or update drift issue
153-
if: matrix.sdk.has_drift == 'true' && matrix.sdk.build_failed == 'true'
124+
if: steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'true'
154125
env:
155126
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
156-
DRIFT_REPORT: ${{ matrix.sdk.drift_report }}
157-
BUILD_REPORT: ${{ matrix.sdk.build_report }}
158127
run: |
159128
RESULT=$(uv run python tasks/scripts/sdk_sync.py manage-issue \
160-
--drift-report "${DRIFT_REPORT:-{}}" \
161-
--build-report "${BUILD_REPORT:-{}}" \
129+
--drift-report "$(cat report/drift.json)" \
130+
--build-report "$(cat report/build.json)" \
162131
--sdk "${{ matrix.sdk.name }}" \
163132
--repo "$GITHUB_REPOSITORY" \
164133
--label "${{ matrix.sdk.label }}")
@@ -168,9 +137,8 @@ jobs:
168137
echo "::error::Issue management for ${{ matrix.sdk.name }} failed: $ACTION"
169138
exit 1
170139
fi
171-
172140
- name: Close resolved drift issue
173-
if: matrix.sdk.has_drift == 'false' || (matrix.sdk.has_drift == 'true' && matrix.sdk.build_failed == 'false')
141+
if: steps.status.outputs.has_drift == 'false' || (steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'false')
174142
env:
175143
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
176144
run: |

‎tasks/go.toml‎

Lines changed: 2 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -138,110 +138,13 @@ hide = true
138138
["go:proto:check"]
139139
description = "Verify generated Go SDK proto files are up to date"
140140
dir = "sdk/go"
141-
run = """
142-
#!/usr/bin/env bash
143-
set -euo pipefail
144-
145-
SDK_ROOT=$(pwd -P)
146-
REPO_ROOT=$(cd ../.. && pwd -P)
147-
148-
for tool in buf protoc-gen-go protoc-gen-go-grpc; do
149-
if ! command -v "$tool" &>/dev/null; then
150-
echo "ERROR: $tool not found. Run 'mise install' to install it."
151-
exit 1
152-
fi
153-
done
154-
155-
WORK_DIR=$(mktemp -d)
156-
trap 'rm -rf "$WORK_DIR"' EXIT
157-
158-
if find proto -maxdepth 1 -name '*.proto' -print -quit | grep -q .; then
159-
echo "ERROR: sdk/go/proto contains copied proto sources."
160-
echo "Proto sources belong in the repository root proto/ directory."
161-
exit 1
162-
fi
163-
164-
# Generate to temp directory with adjusted output path
165-
CHECK_TEMPLATE=$(sed 's|out: sdk/go|out: '"$WORK_DIR"'|' buf.gen.yaml)
166-
(cd "$REPO_ROOT" && buf generate --template "$CHECK_TEMPLATE")
167-
168-
DIFF_OUTPUT=$(diff -r "$WORK_DIR/proto" "$SDK_ROOT/proto" 2>&1) || true
169-
170-
if [ -n "$DIFF_OUTPUT" ]; then
171-
echo "ERROR: Generated proto files are out of date."
172-
echo "Run 'mise run go:proto:gen' to regenerate."
173-
echo ""
174-
echo "$DIFF_OUTPUT"
175-
exit 1
176-
fi
177-
178-
echo "Proto check passed: generated files are up to date."
179-
"""
141+
run = 'bash ../../tasks/scripts/go_proto_check.sh text'
180142
hide = true
181143

182144
["go:proto:drift"]
183145
description = "Check Go SDK proto drift and output a JSON report"
184146
dir = "sdk/go"
185-
run = """
186-
#!/usr/bin/env bash
187-
set -euo pipefail
188-
189-
SDK_ROOT=$(pwd -P)
190-
REPO_ROOT=$(cd ../.. && pwd -P)
191-
192-
for tool in buf protoc-gen-go protoc-gen-go-grpc jq; do
193-
if ! command -v "$tool" &>/dev/null; then
194-
echo '{"sdk":"go","synced":false,"error":"'"$tool"' not found"}'
195-
exit 1
196-
fi
197-
done
198-
199-
WORK_DIR=$(mktemp -d)
200-
trap 'rm -rf "$WORK_DIR"' EXIT
201-
202-
CHECK_TEMPLATE=$(sed 's|out: sdk/go|out: '"$WORK_DIR"'|' buf.gen.yaml)
203-
if ! (cd "$REPO_ROOT" && buf generate --template "$CHECK_TEMPLATE") >/dev/null 2>&1; then
204-
jq -n -c --arg sdk "go" '{sdk:$sdk, synced:false, files:[], summary:"buf generate failed"}'
205-
exit 1
206-
fi
207-
208-
NDJSON_FILE=$(mktemp)
209-
210-
for f in $(find "$WORK_DIR/proto" -name '*.go' -type f | sort); do
211-
REL=${f#"$WORK_DIR/proto/"}
212-
COMMITTED="$SDK_ROOT/proto/$REL"
213-
214-
if [ ! -f "$COMMITTED" ]; then
215-
printf '%s\t%s\t%s\n' "$REL" "added" "0" >> "$NDJSON_FILE"
216-
else
217-
DIFF_LINES=$(diff -u "$COMMITTED" "$f" 2>/dev/null | wc -l | tr -d ' ') || true
218-
if [ "$DIFF_LINES" -gt 0 ]; then
219-
printf '%s\t%s\t%s\n' "$REL" "modified" "$DIFF_LINES" >> "$NDJSON_FILE"
220-
fi
221-
fi
222-
done
223-
224-
for f in $(find "$SDK_ROOT/proto" -name '*.go' -type f | sort); do
225-
REL=${f#"$SDK_ROOT/proto/"}
226-
REGEN="$WORK_DIR/proto/$REL"
227-
if [ ! -f "$REGEN" ]; then
228-
printf '%s\t%s\t%s\n' "$REL" "removed" "0" >> "$NDJSON_FILE"
229-
fi
230-
done
231-
232-
jq -R -c -s --arg sdk "go" '
233-
split("\n") | map(select(length > 0) | split("\t") |
234-
{name: .[0], status: .[1], diff_lines: (.[2] | tonumber)}) |
235-
{sdk: $sdk, synced: (length == 0), files: .,
236-
summary: (if length == 0 then "all files synced"
237-
else "\\(length) file(s) drifted" end)}
238-
' "$NDJSON_FILE"
239-
240-
DRIFTED=$(wc -l < "$NDJSON_FILE" | tr -d ' ')
241-
rm -f "$NDJSON_FILE"
242-
243-
[ "$DRIFTED" -eq 0 ] && exit 0 || exit 1
244-
"""
147+
run = 'bash ../../tasks/scripts/go_proto_check.sh json'
245148
hide = true
246149

247150
["go:proto:build-check"]

0 commit comments

Comments
 (0)