Skip to content

Commit 1fdb127

Browse files
committed
feat(collection): add generic collector marketplace
1 parent 204cb62 commit 1fdb127

63 files changed

Lines changed: 2430 additions & 1755 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env.example

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,8 @@ ADMIN_SUBJECT=
2222

2323
# --- Headless Hub ---
2424
# Provision this file on the server before the first deployment. It contains the Hub API key,
25-
# owner sub, managed Subject IDs, and Collector Instance configuration.
25+
# owner sub and infrastructure settings. Collector lifecycle is managed through the Hub UI.
2626
HEADLESS_CONFIG_PATH=./.local/heartbeat-headless.json
27-
# Host directory holding the Collector Packages, mounted read-only at /package-source.
28-
# The Headless image no longer carries any Package: build it separately with
29-
# scripts/build-vrchat-package.sh (or copy the released Package here), and the Hub installs
30-
# it into its own data directory before running it. Config `packageDirectory` points at
31-
# /package-source/vrchat.
32-
HEADLESS_PACKAGE_SOURCE_PATH=./.local/collector-packages
3327

3428
# --- Recap LLM (ADR-023, OpenAI-compatible) ---
3529
# BaseUrl/Model have compose defaults (DeepSeek); override only if switching provider.
@@ -40,8 +34,3 @@ RECAP_API_KEY=sk-change-me
4034
# Cost is real: on a busy day's digest, high delayed the first narrative token ~175s,
4135
# low ~17s, a non-thinking model ~0.5s. Model-dependent; tune after watching your own days.
4236
RECAP_REASONING_EFFORT=high
43-
44-
# --- VRChat Account Collector ---
45-
VRChat__Username=your-vrchat-username
46-
VRChat__Password=your-vrchat-password
47-
VRChat__TwoFactorSecret=

.env.local.example

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,9 @@ AUTH_OIDC_AUDIENCE=
1717
# 部署管理员的 JWT sub。允许调用全局 App 映射管理 API;普通采集与查询不依赖它。
1818
ADMIN_SUBJECT=
1919

20-
# 无头 Hub 是本地栈默认服务。先把 compose 示例复制到这个位置并填入 API key、owner sub
21-
# 与 Subject ID;若配置放在其他位置,改成对应路径
20+
# 无头 Hub 是本地栈默认服务。先把 compose 示例复制到这个位置并填入 API key、owner sub
21+
# Collector 从 Hub 管理页安装,不写在这个配置文件里
2222
HEADLESS_CONFIG_PATH=./.local/heartbeat-headless.json
23-
# 宿主上的 Collector Package 根目录,以只读方式挂到容器的 /package-source。
24-
# Headless image 里不再带 Package:先跑 scripts/build-vrchat-package.sh 单独构建,
25-
# Hub 启动时会把它安装到自己的数据目录再运行。配置里的 packageDirectory 指向 /package-source/vrchat。
26-
HEADLESS_PACKAGE_SOURCE_PATH=./.local/collector-packages
27-
# 本地自动化/离线 E2E 使用 mock;真实账号人工 smoke 时改为 0。
28-
HEARTBEAT_VRCHAT_MOCK=1
2923

3024
# Recap LLM(可选:不填则 /recaps/daily 返回 502"未配置",其余功能不受影响)
3125
RECAP_API_KEY=

.github/workflows/deploy-backend.yml

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
name: Deploy Backend
22

33
on:
4-
# Production stays manual while the breaking Local Calendar Window contract requires a
5-
# maintenance-window cutover. Headless has a separate, still-unimplemented deploy gap.
4+
# Production deployment is explicit and independent from Frontend, Hub, and Collectors.
65
workflow_dispatch:
76

87
env:
@@ -24,15 +23,6 @@ jobs:
2423
- name: Test
2524
run: dotnet test server/Heartbeat.Server.Tests/Heartbeat.Server.Tests.csproj --configuration Release
2625

27-
- name: Test Hub runtime
28-
run: dotnet test collection/hub/Heartbeat.Collection.Hub.Tests/Heartbeat.Collection.Hub.Tests.csproj --configuration Release
29-
30-
- name: Test headless Hub
31-
run: dotnet test collection/hub/Heartbeat.Collection.Headless.Tests/Heartbeat.Collection.Headless.Tests.csproj --configuration Release
32-
33-
- name: Test VRChat Collector
34-
run: dotnet test collection/collectors/Heartbeat.Collector.VRChat.Tests/Heartbeat.Collector.VRChat.Tests.csproj --configuration Release
35-
3626
- uses: docker/setup-buildx-action@v3
3727

3828
- name: Log in to GHCR

.github/workflows/deploy-frontend.yml

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
name: Deploy Frontend
22

33
on:
4-
# Production stays manual while Frontend and Analytics must replace the Local Calendar Window
5-
# contract together. PR/main verification lives in collector-contracts.yml.
4+
# Production deployment is explicit and independent from Backend, Hub, and Collectors.
65
workflow_dispatch:
76

87
env:
@@ -60,14 +59,26 @@ jobs:
6059
runs-on: ubuntu-latest
6160
concurrency: deploy-frontend-production
6261
steps:
62+
- uses: actions/checkout@v4
63+
64+
- name: Sync compose.yml to server
65+
uses: appleboy/scp-action@v1
66+
with:
67+
host: ${{ secrets.SERVER_HOST }}
68+
username: ${{ secrets.SERVER_USER }}
69+
key: ${{ secrets.SERVER_SSH_KEY }}
70+
source: compose.yml
71+
target: /srv/heartbeat
72+
6373
- name: Pull and restart frontend container
6474
uses: appleboy/ssh-action@v1
6575
with:
6676
host: ${{ secrets.SERVER_HOST }}
6777
username: ${{ secrets.SERVER_USER }}
6878
key: ${{ secrets.SERVER_SSH_KEY }}
6979
script: |
80+
set -eu
7081
cd /srv/heartbeat
7182
docker compose pull frontend
72-
docker compose up -d frontend
83+
docker compose up -d --no-deps frontend
7384
docker compose ps frontend

.github/workflows/deploy-hub.yml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Deploy Hub
2+
3+
on:
4+
workflow_dispatch:
5+
6+
env:
7+
IMAGE: ghcr.io/shenxianovo/heartbeat-headless
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: read
14+
packages: write
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- uses: actions/setup-dotnet@v4
19+
with:
20+
dotnet-version: '10.0.x'
21+
22+
- name: Test shared Collector runtime
23+
run: dotnet test collection/hub/Heartbeat.Collection.Hub.Tests/Heartbeat.Collection.Hub.Tests.csproj --configuration Release
24+
25+
- name: Test Headless Hub adapter
26+
run: dotnet test collection/hub/Heartbeat.Collection.Headless.Tests/Heartbeat.Collection.Headless.Tests.csproj --configuration Release
27+
28+
- uses: docker/setup-buildx-action@v3
29+
30+
- name: Log in to GHCR
31+
uses: docker/login-action@v3
32+
with:
33+
registry: ghcr.io
34+
username: ${{ github.actor }}
35+
password: ${{ secrets.GITHUB_TOKEN }}
36+
37+
- name: Build and push
38+
uses: docker/build-push-action@v6
39+
with:
40+
context: .
41+
file: collection/hub/Heartbeat.Collection.Headless/Dockerfile
42+
push: true
43+
tags: |
44+
${{ env.IMAGE }}:latest
45+
${{ env.IMAGE }}:sha-${{ github.sha }}
46+
cache-from: type=gha
47+
cache-to: type=gha,mode=max
48+
49+
deploy:
50+
needs: build
51+
runs-on: ubuntu-latest
52+
concurrency: deploy-hub-production
53+
steps:
54+
- uses: actions/checkout@v4
55+
56+
- name: Sync compose.yml to server
57+
uses: appleboy/scp-action@v1
58+
with:
59+
host: ${{ secrets.SERVER_HOST }}
60+
username: ${{ secrets.SERVER_USER }}
61+
key: ${{ secrets.SERVER_SSH_KEY }}
62+
source: compose.yml
63+
target: /srv/heartbeat
64+
65+
- name: Pull and restart only the Hub container
66+
uses: appleboy/ssh-action@v1
67+
with:
68+
host: ${{ secrets.SERVER_HOST }}
69+
username: ${{ secrets.SERVER_USER }}
70+
key: ${{ secrets.SERVER_SSH_KEY }}
71+
script: |
72+
set -eu
73+
cd /srv/heartbeat
74+
docker compose pull headless
75+
docker compose up -d --no-deps headless
76+
docker compose ps headless

.github/workflows/release-collector-vrchat.yml

Lines changed: 140 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,13 @@ jobs:
8585
artifact_name="$PACKAGE_ID-$version-linux-x64.zip"
8686
test -f "release-output/$artifact_name"
8787
test -f release-output/release.json
88+
test -f release-output/catalog-entry.json
8889
test "$(jq -r .packageId release-output/release.json)" = "$PACKAGE_ID"
8990
test "$(jq -r .version release-output/release.json)" = "$version"
9091
test "$(jq -r .artifact.fileName release-output/release.json)" = "$artifact_name"
92+
test "$(jq -r .packageId release-output/catalog-entry.json)" = "$PACKAGE_ID"
93+
test "$(jq -r '.latest | length' release-output/catalog-entry.json)" = 1
94+
test "$(jq -r '.latest[0].version' release-output/catalog-entry.json)" = "$version"
9195
9296
mkdir extracted
9397
unzip -q "release-output/$artifact_name" -d extracted
@@ -158,18 +162,18 @@ jobs:
158162
source="$staging/release-output"
159163
registry='/srv/heartbeat/collector-registry/v1'
160164
versions="$registry/packages/$package_id/versions"
161-
target="$versions/$version"
162-
pending="$versions/.$version.staging-${{ github.run_id }}-${{ github.run_attempt }}"
165+
version_root="$versions/$version"
166+
target="$version_root/linux-x64"
167+
pending="$version_root/.linux-x64.staging-${{ github.run_id }}-${{ github.run_attempt }}"
163168
164169
test -f "$source/$artifact"
165170
test -f "$source/release.json"
166-
install -d -m 0755 -- "$versions"
171+
install -d -m 0755 -- "$version_root"
167172
168173
if [ -d "$target" ]; then
169174
if cmp -s "$source/$artifact" "$target/$artifact" && \
170175
cmp -s "$source/release.json" "$target/release.json"; then
171176
echo "Exact release already exists; treating rerun as idempotent."
172-
rm -rf -- "$staging"
173177
exit 0
174178
fi
175179
echo "Refusing to overwrite $package_id $version with different bytes." >&2
@@ -181,15 +185,14 @@ jobs:
181185
install -m 0644 "$source/$artifact" "$pending/$artifact"
182186
install -m 0644 "$source/release.json" "$pending/release.json"
183187
mv -- "$pending" "$target"
184-
rm -rf -- "$staging"
185188
186189
- name: Verify the public exact release bytes
187190
shell: bash
188191
run: |
189192
set -Eeuo pipefail
190193
version='${{ needs.build.outputs.version }}'
191194
artifact='${{ needs.build.outputs.artifact_name }}'
192-
url="$REGISTRY_BASE_URL/packages/$PACKAGE_ID/versions/$version"
195+
url="$REGISTRY_BASE_URL/packages/$PACKAGE_ID/versions/$version/linux-x64"
193196
curl --fail --silent --show-error --location \
194197
--retry 12 --retry-delay 5 \
195198
"$url/$artifact?release-run=$GITHUB_RUN_ID" \
@@ -200,3 +203,134 @@ jobs:
200203
--output "$RUNNER_TEMP/release.json"
201204
cmp release-output/$artifact "$RUNNER_TEMP/$artifact"
202205
cmp release-output/release.json "$RUNNER_TEMP/release.json"
206+
207+
- name: Advance the generic Catalog after the exact release is public
208+
uses: appleboy/ssh-action@v1
209+
with:
210+
host: ${{ secrets.SERVER_HOST }}
211+
username: ${{ secrets.SERVER_USER }}
212+
key: ${{ secrets.SERVER_SSH_KEY }}
213+
script: |
214+
set -eu
215+
package_id='${{ env.PACKAGE_ID }}'
216+
version='${{ needs.build.outputs.version }}'
217+
staging='/tmp/heartbeat-collector-release-${{ github.run_id }}-${{ github.run_attempt }}'
218+
entry="$staging/release-output/catalog-entry.json"
219+
registry='/srv/heartbeat/collector-registry/v1'
220+
catalog="$registry/catalog.json"
221+
lock="$registry/.catalog.lock"
222+
223+
command -v jq >/dev/null 2>&1
224+
command -v flock >/dev/null 2>&1
225+
test -f "$entry"
226+
test "$(jq -r .packageId "$entry")" = "$package_id"
227+
test "$(jq -r '.latest | length' "$entry")" = 1
228+
test "$(jq -r '.latest[0].version' "$entry")" = "$version"
229+
target_os=$(jq -r '.latest[0].target.os' "$entry")
230+
target_arch=$(jq -r '.latest[0].target.arch' "$entry")
231+
install -d -m 0755 -- "$registry"
232+
233+
(
234+
flock -x 9
235+
current=''
236+
existing=''
237+
if [ -f "$catalog" ]; then
238+
jq -e '
239+
.schemaVersion == 1 and
240+
(.packages | type == "array") and
241+
([.packages[].packageId] | length == (unique | length)) and
242+
all(.packages[];
243+
(.latest | type == "array") and
244+
([.latest[] | "\(.target.os)/\(.target.arch)"] | length == (unique | length)))
245+
' "$catalog" >/dev/null
246+
existing=$(jq -c --arg id "$package_id" \
247+
'.packages[] | select(.packageId == $id)' "$catalog")
248+
current=$(jq -r --arg id "$package_id" --arg os "$target_os" --arg arch "$target_arch" \
249+
'.packages[] | select(.packageId == $id) | .latest[] |
250+
select(.target.os == $os and .target.arch == $arch) | .version' "$catalog")
251+
fi
252+
253+
if [ -n "$current" ] && [ "$current" != "$version" ] && \
254+
[ "$(printf '%s\n%s\n' "$current" "$version" | sort -V | tail -n 1)" = "$current" ]; then
255+
echo "Catalog already points at newer $package_id $current; leaving it unchanged."
256+
exit 0
257+
fi
258+
259+
if [ "$current" = "$version" ]; then
260+
if jq -e --argjson expected "$(cat "$entry")" --arg id "$package_id" \
261+
--arg os "$target_os" --arg arch "$target_arch" '
262+
.packages[] | select(.packageId == $id) as $actual |
263+
$actual.displayName == $expected.displayName and
264+
$actual.summary == $expected.summary and
265+
($actual.latest[] | select(.target.os == $os and .target.arch == $arch)) == $expected.latest[0]
266+
' "$catalog" >/dev/null; then
267+
echo "Catalog entry already exists; treating rerun as idempotent."
268+
exit 0
269+
fi
270+
echo "Refusing to replace $package_id $target_os/$target_arch $version Catalog metadata with different bytes." >&2
271+
exit 1
272+
fi
273+
274+
incoming=$(cat "$entry")
275+
if [ -n "$existing" ]; then
276+
if ! jq -e --argjson existing "$existing" --argjson incoming "$incoming" \
277+
'$existing.displayName == $incoming.displayName and
278+
$existing.summary == $incoming.summary' >/dev/null; then
279+
echo "Refusing to change shared Catalog presentation while other targets may exist." >&2
280+
exit 1
281+
fi
282+
merged=$(jq -n --argjson existing "$existing" --argjson incoming "$incoming" \
283+
--arg os "$target_os" --arg arch "$target_arch" '
284+
$existing | .latest = (
285+
[.latest[] | select(.target.os != $os or .target.arch != $arch)] +
286+
$incoming.latest |
287+
sort_by(.target.os, .target.arch)
288+
)
289+
')
290+
else
291+
merged=$incoming
292+
fi
293+
294+
temporary="$registry/.catalog.json.${{ github.run_id }}-${{ github.run_attempt }}"
295+
if [ -f "$catalog" ]; then
296+
jq --arg id "$package_id" --argjson merged "$merged" \
297+
'.packages = ([.packages[] | select(.packageId != $id)] + [$merged] | sort_by(.packageId))' \
298+
"$catalog" > "$temporary"
299+
else
300+
jq -n --argjson entry "$merged" \
301+
'{schemaVersion: 1, packages: [$entry]}' > "$temporary"
302+
fi
303+
chmod 0644 "$temporary"
304+
mv -- "$temporary" "$catalog"
305+
) 9>"$lock"
306+
307+
- name: Verify the public Catalog and clean server staging
308+
shell: bash
309+
run: |
310+
set -Eeuo pipefail
311+
version='${{ needs.build.outputs.version }}'
312+
curl --fail --silent --show-error --location \
313+
--retry 12 --retry-delay 5 \
314+
"$REGISTRY_BASE_URL/catalog.json?release-run=$GITHUB_RUN_ID" \
315+
--output "$RUNNER_TEMP/catalog.json"
316+
jq -e --arg id "$PACKAGE_ID" --arg version "$version" \
317+
--arg os linux --arg arch x64 \
318+
'.schemaVersion == 1 and
319+
([.packages[] | select(.packageId == $id)] | length) == 1 and
320+
((.packages[] | select(.packageId == $id) | .latest[] |
321+
select(.target.os == $os and .target.arch == $arch) | .version) as $current |
322+
($current == $version or
323+
([ $current, $version ] | sort_by(split(".") | map(tonumber)) | last) == $current))' \
324+
"$RUNNER_TEMP/catalog.json" >/dev/null
325+
326+
- name: Remove isolated server staging
327+
if: always()
328+
uses: appleboy/ssh-action@v1
329+
with:
330+
host: ${{ secrets.SERVER_HOST }}
331+
username: ${{ secrets.SERVER_USER }}
332+
key: ${{ secrets.SERVER_SSH_KEY }}
333+
script: |
334+
set -eu
335+
staging='/tmp/heartbeat-collector-release-${{ github.run_id }}-${{ github.run_attempt }}'
336+
rm -rf -- "$staging"

0 commit comments

Comments
 (0)