|
| 1 | +name: Temporary Hugging Face dataset migration |
| 2 | + |
| 3 | +on: |
| 4 | + push: |
| 5 | + branches: |
| 6 | + - agent/hf-dataset-migration-actions |
| 7 | + |
| 8 | +permissions: |
| 9 | + contents: read |
| 10 | + |
| 11 | +jobs: |
| 12 | + upload: |
| 13 | + name: Upload ${{ matrix.name }} |
| 14 | + runs-on: ubuntu-latest |
| 15 | + timeout-minutes: 360 |
| 16 | + strategy: |
| 17 | + fail-fast: false |
| 18 | + max-parallel: 2 |
| 19 | + matrix: |
| 20 | + include: |
| 21 | + - name: augmented OBB |
| 22 | + path: data/augmented_obb |
| 23 | + repo: SanderGi/pcb-detection-augmented-obb |
| 24 | + - name: augmented segmentation |
| 25 | + path: data/augmented_seg |
| 26 | + repo: SanderGi/pcb-detection-augmented-seg |
| 27 | + |
| 28 | + steps: |
| 29 | + - name: Free runner disk space |
| 30 | + run: | |
| 31 | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL |
| 32 | + df -h / |
| 33 | +
|
| 34 | + - name: Check out source history without LFS payloads |
| 35 | + uses: actions/checkout@v4 |
| 36 | + with: |
| 37 | + fetch-depth: 1 |
| 38 | + lfs: false |
| 39 | + |
| 40 | + - name: Download only this dataset from Git LFS |
| 41 | + env: |
| 42 | + DATASET_PATH: ${{ matrix.path }} |
| 43 | + run: | |
| 44 | + git lfs install --local |
| 45 | + git lfs pull --include="${DATASET_PATH}/**" --exclude="" |
| 46 | +
|
| 47 | + - name: Install current Hugging Face uploader |
| 48 | + run: python -m pip install --upgrade huggingface_hub |
| 49 | + |
| 50 | + - name: Upload dataset |
| 51 | + env: |
| 52 | + HF_TOKEN: ${{ secrets.HF_TOKEN }} |
| 53 | + HF_XET_HIGH_PERFORMANCE: "1" |
| 54 | + DATASET_PATH: ${{ matrix.path }} |
| 55 | + HF_REPO: ${{ matrix.repo }} |
| 56 | + run: | |
| 57 | + hf upload "$HF_REPO" "$DATASET_PATH" . \ |
| 58 | + --repo-type dataset \ |
| 59 | + --exclude '.cache/**' \ |
| 60 | + --commit-message "Complete PCB-Detection dataset migration" |
| 61 | +
|
| 62 | + - name: Verify remote manifest |
| 63 | + env: |
| 64 | + HF_TOKEN: ${{ secrets.HF_TOKEN }} |
| 65 | + DATASET_PATH: ${{ matrix.path }} |
| 66 | + HF_REPO: ${{ matrix.repo }} |
| 67 | + run: | |
| 68 | + python - <<'PY' |
| 69 | + import os |
| 70 | + from pathlib import Path |
| 71 | +
|
| 72 | + from huggingface_hub import HfApi |
| 73 | +
|
| 74 | + root = Path(os.environ["DATASET_PATH"]) |
| 75 | + local = { |
| 76 | + path.relative_to(root).as_posix() |
| 77 | + for path in root.rglob("*") |
| 78 | + if path.is_file() and ".cache" not in path.parts |
| 79 | + } |
| 80 | + info = HfApi().repo_info( |
| 81 | + os.environ["HF_REPO"], repo_type="dataset", files_metadata=True |
| 82 | + ) |
| 83 | + remote = { |
| 84 | + item.rfilename |
| 85 | + for item in info.siblings |
| 86 | + if item.rfilename != ".gitattributes" |
| 87 | + } |
| 88 | + missing = sorted(local - remote) |
| 89 | + extra = sorted(remote - local) |
| 90 | + print( |
| 91 | + f"local={len(local)} remote={len(remote)} " |
| 92 | + f"missing={len(missing)} extra={len(extra)} sha={info.sha}" |
| 93 | + ) |
| 94 | + if missing or extra: |
| 95 | + print("Missing sample:", missing[:20]) |
| 96 | + print("Extra sample:", extra[:20]) |
| 97 | + raise SystemExit("Remote dataset manifest does not match local source") |
| 98 | + PY |
0 commit comments