Skip to content

Commit 7e31af6

Browse files
BlueSkyXNBlueSkyXN
authored
fix(hfs): 固定 Hugging Face 客户端并对齐可选 Secret
HFS v2 clean-rebuild delivery gate; exact-head CI passed.
1 parent 2035445 commit 7e31af6

6 files changed

Lines changed: 113 additions & 31 deletions

File tree

‎.github/workflows/deploy-hf-space.yml‎

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ concurrency:
2626
cancel-in-progress: false
2727

2828
env:
29-
# 1.5.0 predates the Space Settings subcommands used below.
30-
HF_CLI_VERSION: "1.24.0"
29+
HF_CLI_VERSION: "1.5.0"
30+
HF_CLI_CLICK_VERSION: "8.3.3"
3131

3232
jobs:
3333
deploy:
@@ -51,14 +51,28 @@ jobs:
5151
SPACE_ID="$(python3 -c 'import os,tomllib; print(tomllib.load(open(os.environ["HFS_MANIFEST"], "rb"))["space"])')"
5252
echo "SPACE_ID=$SPACE_ID" >> "$GITHUB_ENV"
5353
echo "LIBREFS_COMMIT=$LIBREFS_COMMIT" >> "$GITHUB_ENV"
54-
python3 -m pip install --disable-pip-version-check "huggingface_hub==${HF_CLI_VERSION}"
55-
if [[ "${{ inputs.target }}" == candidate ]]; then
56-
test "$(hf spaces info "$SPACE_ID" --expand private --json | python3 -c 'import json,sys; print(str(json.load(sys.stdin)["private"]).lower())')" = true
57-
fi
58-
configured_variables="$(hf spaces variables list "$SPACE_ID")"
59-
grep -F "LIBREFS_COMMIT" <<<"$configured_variables" >/dev/null
60-
grep -F "$LIBREFS_COMMIT" <<<"$configured_variables" >/dev/null
61-
grep -E 'HFS_RELEASE_BUILD.*true|true.*HFS_RELEASE_BUILD' <<<"$configured_variables" >/dev/null
54+
python3 -m pip install --disable-pip-version-check \
55+
"huggingface_hub==${HF_CLI_VERSION}" \
56+
"click==${HF_CLI_CLICK_VERSION}"
57+
export SPACE_ID LIBREFS_COMMIT
58+
HFS_TARGET="${{ inputs.target }}" python3 - <<'PY'
59+
import os
60+
from huggingface_hub import HfApi
61+
62+
api = HfApi(token=os.environ["HF_TOKEN"])
63+
info = api.space_info(os.environ["SPACE_ID"], expand=["private"])
64+
if os.environ["HFS_TARGET"] == "candidate" and not info.private:
65+
raise SystemExit("candidate Space must be private")
66+
variables = api.get_space_variables(os.environ["SPACE_ID"])
67+
expected = {
68+
"LIBREFS_COMMIT": os.environ["LIBREFS_COMMIT"],
69+
"HFS_RELEASE_BUILD": "true",
70+
}
71+
for name, value in expected.items():
72+
remote = variables.get(name)
73+
if remote is None or remote.value != value:
74+
raise SystemExit(f"configured Space Variable mismatch: {name}")
75+
PY
6276
6377
- name: Export and verify the allowlisted Space wrapper
6478
env:
@@ -80,7 +94,7 @@ jobs:
8094
bundle_dir="$RUNNER_TEMP/librefs-hfs-space-bundle"
8195
preflight_dir="$RUNNER_TEMP/librefs-hfs-space-preflight"
8296
expected_paths="$(cd "$bundle_dir" && find . -type f -print | LC_ALL=C sort | cut -c3-)"
83-
if ! hf download "$SPACE_ID" --repo-type space --local-dir "$preflight_dir" >/dev/null 2>&1; then
97+
if ! python3 -m huggingface_hub.cli.hf download "$SPACE_ID" --repo-type space --local-dir "$preflight_dir" >/dev/null 2>&1; then
8498
echo "Refusing to upload: unable to read the current Space repository." >&2
8599
exit 1
86100
fi
@@ -101,7 +115,7 @@ jobs:
101115
HF_TOKEN: ${{ secrets.HF_TOKEN }}
102116
run: |
103117
set -Eeuo pipefail
104-
hf upload "$SPACE_ID" "$RUNNER_TEMP/librefs-hfs-space-bundle" . \
118+
python3 -m huggingface_hub.cli.hf upload "$SPACE_ID" "$RUNNER_TEMP/librefs-hfs-space-bundle" . \
105119
--repo-type space \
106120
--commit-message "Deploy verified wrapper ${GITHUB_SHA}"
107121
@@ -113,7 +127,7 @@ jobs:
113127
bundle_dir="$RUNNER_TEMP/librefs-hfs-space-bundle"
114128
readback_dir="$RUNNER_TEMP/librefs-hfs-space-readback"
115129
expected_paths="$(cd "$bundle_dir" && find . -type f -print | LC_ALL=C sort | cut -c3-)"
116-
if ! hf download "$SPACE_ID" --repo-type space --local-dir "$readback_dir" >/dev/null 2>&1; then
130+
if ! python3 -m huggingface_hub.cli.hf download "$SPACE_ID" --repo-type space --local-dir "$readback_dir" >/dev/null 2>&1; then
117131
echo "Unable to read back the uploaded Space repository." >&2
118132
exit 1
119133
fi
@@ -125,7 +139,17 @@ jobs:
125139
cd "$readback_dir"
126140
sha256sum -c SHA256SUMS
127141
)
128-
hf spaces info "$SPACE_ID"
142+
python3 - <<'PY'
143+
import os
144+
import re
145+
from huggingface_hub import HfApi
146+
147+
space_id = os.environ["SPACE_ID"]
148+
info = HfApi(token=os.environ["HF_TOKEN"]).space_info(space_id)
149+
if info.id != space_id or not re.fullmatch(r"[0-9a-f]{40}", info.sha or ""):
150+
raise SystemExit("Space metadata readback returned unexpected identity or revision")
151+
print(f"Space metadata readback succeeded: id={info.id}, sha={info.sha}")
152+
PY
129153
130154
- name: Record remaining runtime owner gates
131155
run: |

‎docs/operations.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,10 @@ scripts/sample-hf-bucket-storage.sh \
193193

194194
| 字段 | 含义 |
195195
| --- | --- |
196-
| `info_size` | `hf buckets info` 返回的 bucket 账面 size。 |
197-
| `info_total_files` | `hf buckets info` 返回的 total files。 |
198-
| `visible_sum` | `hf buckets list -R` 当前 visible file size 求和。 |
199-
| `visible_files` | `hf buckets list -R` 当前 visible file 数。 |
196+
| `info_size` | pinned `HfApi.bucket_info()` 返回的 bucket 账面 size。 |
197+
| `info_total_files` | pinned `HfApi.bucket_info()` 返回的 total files。 |
198+
| `visible_sum` | pinned module CLI `buckets list --recursive --format json` 当前 visible file size 求和。 |
199+
| `visible_files` | pinned module CLI `buckets list --recursive --format json` 当前 visible file 数。 |
200200
| `drift_bytes` | `info_size - visible_sum`。 |
201201
| `ops_visible_sum` | 可选 `/_ops/storage` 返回的容器内 visible bytes。 |
202202
| `largest_visible_file` | HF visible tree 中最大的当前文件。 |

‎docs/source-walkthrough.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ ops 鉴权支持三种方式：
427427
- `hfs/start.sh` 的 Secret 校验、公开 URL 推导、Console redirect URL、ops/admin service 和 Nginx 配置预检。
428428
- `hfs/nginx.conf` 的 `7860` 监听、`/_ops/`、`/_admin/`、`/console/` 子路径、S3 根路径和 iframe header。
429429
- `hfs/ops_service.py` 的 `/_ops/storage` 路由和 storage metrics。
430-
- `scripts/sample-hf-bucket-storage.sh` 的存在性、可执行位、Bash 语法和 `hf`/`python3` 只读采样约束。
430+
- `scripts/sample-hf-bucket-storage.sh` 的存在性、可执行位、Bash 语法和 pinned `HfApi`/module CLI 只读采样约束。
431431
- `LICENSE` 是否仍是 AGPL-3.0。
432432
- 如果本机安装了 `nginx`，运行 `nginx -t -c "$PWD/hfs/nginx.conf"`。
433433

@@ -465,10 +465,10 @@ scripts/sample-hf-bucket-storage.sh \
465465
--ops-url https://blueskyxn-librefs-hfs.hf.space/_ops
466466
```
467467

468-
脚本会调用：
468+
脚本固定验证 `huggingface_hub==1.5.0` 与 `click==8.3.3`，然后调用：
469469

470-
- `hf buckets info "$bucket" --json`
471-
- `hf buckets list "$bucket" -R --json`
470+
- `HfApi().bucket_info("$bucket")`，生成只包含 bucket metadata 的结构化 JSON
471+
- `python3 -m huggingface_hub.cli.hf buckets list "$bucket" --recursive --format json`
472472
- 可选 `curl -H "X-Ops-Token: $OPS_TOKEN" "$ops_url/storage?format=json"`
473473

474474
输出字段包括 `timestamp_utc`、`info_size`、`info_total_files`、`visible_sum`、`visible_files`、`drift_bytes`、`ops_visible_sum` 和 `largest_visible_file`。脚本用 `python3` 解析 JSON，不依赖 `jq`，也不会打印 `OPS_TOKEN`。

‎hfs-dev.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ secrets = [
2828
"MINIO_ROOT_USER",
2929
"MINIO_ROOT_PASSWORD",
3030
"OPS_TOKEN",
31+
]
32+
optional_secrets = [
3133
"ADMIN_TOKEN",
3234
]
3335
variables = [

‎scripts/sample-hf-bucket-storage.sh‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
set -Eeuo pipefail
33

44
DEFAULT_BUCKET="BlueSkyXN/librefs-hfs-data"
5+
EXPECTED_HF_HUB_VERSION="1.5.0"
6+
EXPECTED_CLICK_VERSION="8.3.3"
57
BUCKET="${HF_BUCKET_ID:-$DEFAULT_BUCKET}"
68
OPS_URL="${OPS_URL:-}"
79
INTERVAL=0
@@ -83,13 +85,28 @@ if [[ "$COUNT" == "0" && "$INTERVAL" == "0" ]]; then
8385
exit 2
8486
fi
8587

86-
if ! command -v hf >/dev/null 2>&1; then
87-
echo "hf CLI is required." >&2
88+
if ! command -v python3 >/dev/null 2>&1; then
89+
echo "python3 is required." >&2
8890
exit 1
8991
fi
9092

91-
if ! command -v python3 >/dev/null 2>&1; then
92-
echo "python3 is required." >&2
93+
if ! python3 - "$EXPECTED_HF_HUB_VERSION" "$EXPECTED_CLICK_VERSION" <<'PY'
94+
import importlib.metadata
95+
import sys
96+
97+
expected_hub, expected_click = sys.argv[1:]
98+
try:
99+
actual_hub = importlib.metadata.version("huggingface_hub")
100+
actual_click = importlib.metadata.version("click")
101+
except importlib.metadata.PackageNotFoundError as exc:
102+
raise SystemExit(f"required Python package is missing: {exc.name}") from exc
103+
if actual_hub != expected_hub or actual_click != expected_click:
104+
raise SystemExit(
105+
"storage sampler requires "
106+
f"huggingface_hub=={expected_hub} and click=={expected_click}"
107+
)
108+
PY
109+
then
93110
exit 1
94111
fi
95112

@@ -124,8 +141,23 @@ sample_once() {
124141
local list_json="$TMP_DIR/list.json"
125142
local ops_json="$TMP_DIR/ops.json"
126143

127-
hf buckets info "$BUCKET" --json >"$info_json"
128-
hf buckets list "$BUCKET" -R --json >"$list_json"
144+
python3 - "$BUCKET" >"$info_json" <<'PY'
145+
import json
146+
import sys
147+
148+
from huggingface_hub import HfApi
149+
150+
info = HfApi().bucket_info(sys.argv[1])
151+
print(json.dumps({
152+
"id": info.id,
153+
"private": info.private,
154+
"createdAt": info.created_at.isoformat(),
155+
"size": info.size,
156+
"totalFiles": info.total_files,
157+
}, sort_keys=True))
158+
PY
159+
python3 -m huggingface_hub.cli.hf buckets list \
160+
"$BUCKET" --recursive --format json >"$list_json"
129161

130162
if [[ -n "$OPS_URL" ]]; then
131163
curl -fsS -H "X-Ops-Token: ${OPS_TOKEN}" "$(ops_endpoint)" >"$ops_json"

‎scripts/validate-contract.sh‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ require_pattern hfs-dev.toml '^runtime_mode = "bundle-only-build"$' 'runtime mus
9999
require_pattern hfs-dev.toml '^release_commit_env = "LIBREFS_COMMIT"$' 'registry must name the upstream release commit pin'
100100
require_pattern hfs-dev.toml '^release_gate_env = "HFS_RELEASE_BUILD"$' 'registry must name the release gate'
101101
require_pattern hfs-dev.toml '"MINIO_ROOT_PASSWORD"' 'registry must name required Space secrets without values'
102+
require_pattern hfs-dev.toml '^optional_secrets = \[$' 'registry must separate disabled admin credentials as optional'
103+
require_pattern hfs-dev.toml '"ADMIN_TOKEN"' 'registry must keep the disabled admin token registered'
102104
require_pattern hfs-dev.toml '"HF_TOKEN"' 'registry must keep deployment controls local-only'
103105
if grep -Eq '(hf_[A-Za-z0-9]{20,}|ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9]{20,})' hfs-dev.toml; then
104106
echo "Contract check failed: hfs-dev.toml must not contain a credential value" >&2
@@ -204,9 +206,20 @@ require_pattern scripts/smoke-s3-curl.sh 'Authorization: Bearer' 'private candid
204206
require_pattern scripts/smoke-s3-curl.sh 'Refusing to use bucket' 'S3 smoke test must refuse existing buckets'
205207

206208
check "storage sampler script contract" test -f scripts/sample-hf-bucket-storage.sh
207-
require_pattern scripts/sample-hf-bucket-storage.sh 'hf buckets info "\$BUCKET" --json' 'storage sampler must read HF bucket accounting'
208-
require_pattern scripts/sample-hf-bucket-storage.sh 'hf buckets list "\$BUCKET" -R --json' 'storage sampler must read HF bucket visible tree'
209+
require_pattern scripts/sample-hf-bucket-storage.sh '^EXPECTED_HF_HUB_VERSION="1\.5\.0"$' 'storage sampler must pin huggingface_hub 1.5.0'
210+
require_pattern scripts/sample-hf-bucket-storage.sh '^EXPECTED_CLICK_VERSION="8\.3\.3"$' 'storage sampler must pin the module CLI click runtime'
211+
require_pattern scripts/sample-hf-bucket-storage.sh 'HfApi\(\)\.bucket_info\(sys\.argv\[1\]\)' 'storage sampler must use structured HfApi bucket accounting'
212+
require_pattern scripts/sample-hf-bucket-storage.sh 'python3 -m huggingface_hub\.cli\.hf buckets list' 'storage sampler must use the pinned module CLI for the visible tree'
213+
require_pattern scripts/sample-hf-bucket-storage.sh '"\$BUCKET" --recursive --format json' 'storage sampler must request recursive JSON visible-tree output'
209214
require_pattern scripts/sample-hf-bucket-storage.sh 'OPS_TOKEN' 'storage sampler must use OPS_TOKEN only for optional ops endpoint access'
215+
if grep -Eq '(^|[^[:alnum:]_.-])hf[[:space:]]+buckets[[:space:]]+(info|list)([[:space:]]|$)' scripts/sample-hf-bucket-storage.sh; then
216+
echo "Contract check failed: storage sampler must not use the unpinned hf console entrypoint" >&2
217+
exit 1
218+
fi
219+
if grep -Eq 'buckets[[:space:]]+info[^[:cntrl:]]*--(json|format)' scripts/sample-hf-bucket-storage.sh; then
220+
echo "Contract check failed: storage sampler must not claim buckets info has JSON output" >&2
221+
exit 1
222+
fi
210223

211224
check "nginx routing contract" require_pattern hfs/nginx.conf 'listen 7860;' 'Nginx must listen on HF app port 7860'
212225
require_pattern hfs/nginx.conf 'location = /console' 'Nginx must normalize /console'
@@ -241,7 +254,18 @@ require_pattern .github/workflows/deploy-hf-space.yml 'hfs-dev\.candidate\.toml'
241254
require_pattern .github/workflows/deploy-hf-space.yml 'scripts/export-space-bundle\.sh' 'workflow must export the wrapper boundary'
242255
require_pattern .github/workflows/deploy-hf-space.yml 'scripts/verify-space-bundle\.sh' 'workflow must verify the wrapper boundary'
243256
require_pattern .github/workflows/deploy-hf-space.yml 'Refuse a Space repository outside the wrapper boundary' 'workflow must fail closed on legacy Space files'
244-
require_pattern .github/workflows/deploy-hf-space.yml 'hf download' 'workflow must read back the Space repository'
257+
require_pattern .github/workflows/deploy-hf-space.yml 'huggingface_hub\.cli\.hf download' 'workflow must read back the Space repository through the pinned module CLI'
258+
require_pattern .github/workflows/deploy-hf-space.yml 'huggingface_hub\.cli\.hf upload' 'workflow must upload the Space repository through the pinned module CLI'
259+
require_pattern .github/workflows/deploy-hf-space.yml 'HF_CLI_VERSION: "1\.5\.0"' 'workflow must pin huggingface_hub 1.5.0'
260+
require_pattern .github/workflows/deploy-hf-space.yml 'HF_CLI_CLICK_VERSION: "8\.3\.3"' 'workflow must pin the module CLI click runtime'
261+
require_pattern .github/workflows/deploy-hf-space.yml 'huggingface_hub==\$\{HF_CLI_VERSION\}' 'workflow must install the pinned huggingface_hub version'
262+
require_pattern .github/workflows/deploy-hf-space.yml 'click==\$\{HF_CLI_CLICK_VERSION\}' 'workflow must install the pinned click version'
263+
require_pattern .github/workflows/deploy-hf-space.yml 'get_space_variables' 'workflow must read Space Variables through HfApi because the pinned CLI has no Settings subcommands'
264+
require_pattern .github/workflows/deploy-hf-space.yml 'space_info' 'workflow must read Space metadata through HfApi'
265+
if grep -Eq '(^|[^[:alnum:]_.-])hf[[:space:]]+(download|upload|spaces)([[:space:]]|$)' .github/workflows/deploy-hf-space.yml; then
266+
echo "Contract check failed: deployment workflow must invoke the pinned HF CLI through its Python module entrypoint" >&2
267+
exit 1
268+
fi
245269
require_pattern .github/workflows/deploy-hf-space.yml 'sha256sum -c SHA256SUMS' 'workflow must verify complete uploaded wrapper bytes'
246270
if grep -Eq 'git push|--force|--delete|\|\| true' .github/workflows/deploy-hf-space.yml; then
247271
echo "Contract check failed: deployment workflow must not force-push, delete, or bypass a failed check" >&2

0 commit comments

Comments
 (0)