feat(array): builders skip child canonicalization and append nested v… #151
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs after every commit to `develop` (or in other words, _after_ every pull request merges). | |
| name: Develop Benchmarks | |
| on: | |
| push: | |
| branches: [develop] | |
| permissions: | |
| id-token: write # enables AWS-GitHub OIDC | |
| actions: read | |
| contents: write | |
| jobs: | |
| commit-metadata: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - name: Setup AWS CLI | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 | |
| with: | |
| role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole | |
| aws-region: us-east-1 | |
| - name: Upload Commit Metadata | |
| shell: bash | |
| run: | | |
| set -Eeu -o pipefail -x | |
| sudo apt-get update && sudo apt-get install -y jq | |
| bash scripts/commit-json.sh > new-commit.json | |
| bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json | |
| build: | |
| timeout-minutes: 60 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=bench-dedicated/family=c6id.8xlarge/extras=s3-cache/tag=build-{1}', github.run_id, matrix.benchmark.id) | |
| || 'ubuntu-latest' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| benchmark: | |
| - id: random-access-bench | |
| build_args: "--features lance" | |
| - id: compress-bench | |
| build_args: "--features lance" | |
| - id: string-bench | |
| build_args: "" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} | |
| - name: Build binary | |
| shell: bash | |
| env: | |
| RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" | |
| run: | | |
| cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings | |
| - name: Pre-upload benchmark debuginfo to Polar Signals | |
| uses: ./.github/actions/upload-parca-debuginfo | |
| continue-on-error: true | |
| with: | |
| paths: | | |
| target/release_debug/${{ matrix.benchmark.id }} | |
| polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} | |
| project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" | |
| - name: Upload benchmark binary | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: ${{ matrix.benchmark.id }}-binary | |
| path: target/release_debug/${{ matrix.benchmark.id }} | |
| bench: | |
| needs: build | |
| if: ${{ !cancelled() && needs.build.result == 'success' }} | |
| timeout-minutes: 120 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) | |
| || 'ubuntu-latest' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| benchmark: | |
| - id: random-access-bench | |
| name: Random Access | |
| build_args: "--features lance" | |
| v4_ingest: true | |
| - id: compress-bench | |
| name: Compression | |
| build_args: "--features lance" | |
| run_args: "--formats parquet,lance,vortex --ingest-jsonl results.ingest.jsonl" | |
| v4_ingest: true | |
| # No run_args: the default suite emits the three tracked metrics | |
| # (size, write, read). The codec microbenchmark is a local diagnostic. | |
| - id: string-bench | |
| name: String Encoding | |
| v4_ingest: false | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - name: Install DuckDB | |
| uses: ./.github/actions/setup-duckdb | |
| - uses: ./.github/actions/system-info | |
| - name: Download benchmark binary | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: ${{ matrix.benchmark.id }}-binary | |
| path: target/release_debug/ | |
| - name: Make benchmark binary executable | |
| shell: bash | |
| run: chmod +x target/release_debug/${{ matrix.benchmark.id }} | |
| - name: Setup Polar Signals | |
| uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 | |
| # Profiling is observability only, so a flaky agent download must not fail the benchmark. | |
| continue-on-error: true | |
| with: | |
| polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} | |
| labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ matrix.benchmark.id }}" | |
| project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" | |
| job_name: "${{ matrix.benchmark.id }}" | |
| profiling_frequency: 199 | |
| extra_args: "--debuginfo-strip=false" | |
| parca_agent_version: "0.49.0" | |
| - name: Setup benchmark environment | |
| run: sudo bash scripts/setup-benchmark.sh | |
| - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) | |
| if: matrix.benchmark.id == 'random-access-bench' | |
| shell: bash | |
| env: | |
| RUST_BACKTRACE: full | |
| VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" | |
| FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" | |
| run: | | |
| python3 scripts/random-access-split.py --emit-ingest-records | |
| - name: Run ${{ matrix.benchmark.name }} benchmark | |
| if: matrix.benchmark.id != 'random-access-bench' | |
| shell: bash | |
| env: | |
| RUST_BACKTRACE: full | |
| VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" | |
| FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" | |
| run: | | |
| bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \ | |
| ${{ matrix.benchmark.run_args }} -d gh-json -o results.json | |
| - name: Setup AWS CLI | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 | |
| with: | |
| role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole | |
| aws-region: us-east-1 | |
| - name: Upload Benchmark Results | |
| shell: bash | |
| run: | | |
| bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json | |
| # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live | |
| # benchmarks website for suites that emit v4 records; a failure here fails the job. | |
| # All suites also upload S3 results above. string-bench does not emit | |
| # results.ingest.jsonl, so these steps are disabled for it. | |
| # Also gated on the ingest-role ARN var (the assume-role input that MUST exist for | |
| # OIDC to succeed). post-ingest.py mints the RDS IAM token internally (boto3) from | |
| # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert. | |
| # | |
| # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only | |
| # the uv binary, never the synced workspace. A full sync would rebuild | |
| # vortex-python via sccache->S3, which fails under the ingest-role creds | |
| # (rds-db:connect only) and is pure waste here. | |
| - name: Install uv for v4 ingest | |
| if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest | |
| uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 | |
| with: | |
| sync: false | |
| - name: Configure AWS credentials for v4 ingest (OIDC) | |
| if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 | |
| with: | |
| role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} | |
| aws-region: ${{ vars.RDS_BENCH_REGION }} | |
| - name: Ingest results to v4 Postgres | |
| if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest | |
| shell: bash | |
| env: | |
| RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} | |
| RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} | |
| AWS_REGION: ${{ vars.RDS_BENCH_REGION }} | |
| BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }} | |
| BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} | |
| run: | | |
| set -Eeuo pipefail | |
| curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ | |
| -o "${RUNNER_TEMP}/rds-global-bundle.pem" | |
| DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" | |
| uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \ | |
| scripts/post-ingest.py results.ingest.jsonl \ | |
| --postgres "${DSN}" \ | |
| --commit-sha "${{ github.sha }}" \ | |
| --region "${AWS_REGION}" | |
| - name: Alert incident.io | |
| if: failure() | |
| uses: ./.github/actions/alert-incident-io | |
| with: | |
| api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} | |
| alert-title: "${{ matrix.benchmark.name }} benchmark failed on develop" | |
| deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure | |
| sql: | |
| uses: ./.github/workflows/sql-bench-matrix.yml | |
| secrets: inherit | |
| with: | |
| mode: "develop" | |
| matrix_preset: "develop" |