Skip to content

Commit 106de70

Browse files
committed
Add temporary Hugging Face migration workflow
1 parent e6c3ac1 commit 106de70

1 file changed

Lines changed: 98 additions & 0 deletions

File tree

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: Temporary Hugging Face dataset migration
2+
3+
on:
4+
push:
5+
branches:
6+
- agent/hf-dataset-migration-actions
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
upload:
13+
name: Upload ${{ matrix.name }}
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 360
16+
strategy:
17+
fail-fast: false
18+
max-parallel: 2
19+
matrix:
20+
include:
21+
- name: augmented OBB
22+
path: data/augmented_obb
23+
repo: SanderGi/pcb-detection-augmented-obb
24+
- name: augmented segmentation
25+
path: data/augmented_seg
26+
repo: SanderGi/pcb-detection-augmented-seg
27+
28+
steps:
29+
- name: Free runner disk space
30+
run: |
31+
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
32+
df -h /
33+
34+
- name: Check out source history without LFS payloads
35+
uses: actions/checkout@v4
36+
with:
37+
fetch-depth: 1
38+
lfs: false
39+
40+
- name: Download only this dataset from Git LFS
41+
env:
42+
DATASET_PATH: ${{ matrix.path }}
43+
run: |
44+
git lfs install --local
45+
git lfs pull --include="${DATASET_PATH}/**" --exclude=""
46+
47+
- name: Install current Hugging Face uploader
48+
run: python -m pip install --upgrade huggingface_hub
49+
50+
- name: Upload dataset
51+
env:
52+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
53+
HF_XET_HIGH_PERFORMANCE: "1"
54+
DATASET_PATH: ${{ matrix.path }}
55+
HF_REPO: ${{ matrix.repo }}
56+
run: |
57+
hf upload "$HF_REPO" "$DATASET_PATH" . \
58+
--repo-type dataset \
59+
--exclude '.cache/**' \
60+
--commit-message "Complete PCB-Detection dataset migration"
61+
62+
- name: Verify remote manifest
63+
env:
64+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
65+
DATASET_PATH: ${{ matrix.path }}
66+
HF_REPO: ${{ matrix.repo }}
67+
run: |
68+
python - <<'PY'
69+
import os
70+
from pathlib import Path
71+
72+
from huggingface_hub import HfApi
73+
74+
root = Path(os.environ["DATASET_PATH"])
75+
local = {
76+
path.relative_to(root).as_posix()
77+
for path in root.rglob("*")
78+
if path.is_file() and ".cache" not in path.parts
79+
}
80+
info = HfApi().repo_info(
81+
os.environ["HF_REPO"], repo_type="dataset", files_metadata=True
82+
)
83+
remote = {
84+
item.rfilename
85+
for item in info.siblings
86+
if item.rfilename != ".gitattributes"
87+
}
88+
missing = sorted(local - remote)
89+
extra = sorted(remote - local)
90+
print(
91+
f"local={len(local)} remote={len(remote)} "
92+
f"missing={len(missing)} extra={len(extra)} sha={info.sha}"
93+
)
94+
if missing or extra:
95+
print("Missing sample:", missing[:20])
96+
print("Extra sample:", extra[:20])
97+
raise SystemExit("Remote dataset manifest does not match local source")
98+
PY

0 commit comments

Comments
 (0)