Repository navigation
[Spark] Lightweight compaction for clustered tables #72
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Delta Spark" | |
| on: | |
| push: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| pull_request: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| workflow_dispatch: | |
| inputs: | |
| source-cache-key-suffix: | |
| description: Optional suffix to force a distinct Spark Maven cache key | |
| required: false | |
| type: string | |
| # Every PR runs: | |
| # 1. generate-matrix: split Spark versions (from CrossSparkVersions.scala) into published | |
| # vs source-build lists. | |
| # 2. prepare_spark_source_cache: for each source-build Spark version, build Spark once and | |
| # save the Maven cache (via the prepare-spark-env action with build-on-miss). This job is | |
| # non-blocking; any source Spark build/cache failure is reported through source_spark_test. | |
| # 3. test: published Spark shards (4.0/4.1) without waiting on step 2. | |
| # 4. source_spark_test: source-built Spark shards (4.2 today) after step 2. It still runs if | |
| # prepare_spark_source_cache fails, so the source-built Spark signal stays non-blocking. | |
| # | |
| # test and source_spark_test share two composite actions: | |
| # - .github/actions/prepare-spark-env: installs the JDK and, for source-build rows, | |
| # restores the Spark Maven cache (the prepare-cache job builds/saves it; shards restore). | |
| # - .github/actions/run-delta-spark-tests: restores the SBT cache, publishes the pinned | |
| # Unity Catalog build, and runs the sharded tests. | |
| # The Spark SHA, artifact version, and exact Maven cache key all come from | |
| # project/scripts/get_spark_version_info.py, so they are not hand-assembled in this workflow. | |
| jobs: | |
| # Generate Spark versions matrix from CrossSparkVersions.scala | |
| # This ensures the workflow always uses the versions defined in the build | |
| generate-matrix: | |
| name: "Generate Spark Versions Matrix" | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| spark_versions: ${{ steps.generate.outputs.spark_versions }} | |
| source_build_spark_versions: ${{ steps.generate.outputs.source_build_spark_versions }} | |
| non_source_build_spark_versions: ${{ steps.generate.outputs.non_source_build_spark_versions }} | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: install java | |
| uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 | |
| with: | |
| distribution: "zulu" | |
| java-version: "17" | |
| - name: Generate Spark versions matrix | |
| id: generate | |
| run: | | |
| # The script automatically generates spark-versions.json from CrossSparkVersions.scala | |
| SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --all-spark-versions) | |
| SOURCE_BUILD_SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --source-build-spark-versions) | |
| NON_SOURCE_BUILD_SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --non-source-build-spark-versions) | |
| echo "spark_versions=$SPARK_VERSIONS" >> $GITHUB_OUTPUT | |
| echo "source_build_spark_versions=$SOURCE_BUILD_SPARK_VERSIONS" >> $GITHUB_OUTPUT | |
| echo "non_source_build_spark_versions=$NON_SOURCE_BUILD_SPARK_VERSIONS" >> $GITHUB_OUTPUT | |
| echo "Generated Spark versions: $SPARK_VERSIONS" | |
| echo "Generated source-build Spark versions: $SOURCE_BUILD_SPARK_VERSIONS" | |
| echo "Generated non-source-build Spark versions: $NON_SOURCE_BUILD_SPARK_VERSIONS" | |
| # Builds ~/.m2/repository/org/apache/spark once per source-build Spark version so the | |
| # sharded source_spark_test jobs all restore the same cache instead of rebuilding 8x. | |
| prepare_spark_source_cache: | |
| name: "Prepare Spark ${{ matrix.spark_version }} source cache" | |
| runs-on: ubuntu-24.04 | |
| needs: generate-matrix | |
| # Source-built Spark tracks unreleased APIs. Keep the producer non-blocking too, | |
| # otherwise a checkout/build/cache-save failure blocks the whole workflow before | |
| # source_spark_test can surface the same signal. | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| spark_version: ${{ fromJson(needs.generate-matrix.outputs.source_build_spark_versions) }} | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: Prepare Spark environment (build + cache) | |
| uses: ./.github/actions/prepare-spark-env | |
| with: | |
| spark-version: ${{ matrix.spark_version }} | |
| use-spark-source-cache: 'true' | |
| build-on-miss: 'true' | |
| cache-key-suffix: ${{ github.event.inputs['source-cache-key-suffix'] || '' }} | |
| # Published Spark artifacts only. This job does not depend on prepare_spark_source_cache. | |
| test: | |
| name: "DS: Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}" | |
| runs-on: ubuntu-24.04 | |
| needs: generate-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Spark versions are dynamically generated from CrossSparkVersions.scala | |
| # DO NOT hardcode versions here - they are automatically loaded from the build configuration | |
| spark_version: ${{ fromJson(needs.generate-matrix.outputs.non_source_build_spark_versions) }} | |
| # These Scala versions must match those in the build.sbt | |
| scala: [2.13.16] | |
| # Important: This list of shards must be [0..NUM_SHARDS - 1] | |
| shard: [0, 1, 2, 3, 4, 5, 6, 7] | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: Prepare Spark environment | |
| uses: ./.github/actions/prepare-spark-env | |
| with: | |
| spark-version: ${{ matrix.spark_version }} | |
| - name: Run Delta Spark tests | |
| uses: ./.github/actions/run-delta-spark-tests | |
| with: | |
| spark-version: ${{ matrix.spark_version }} | |
| scala: ${{ matrix.scala }} | |
| shard: ${{ matrix.shard }} | |
| # Important: This must be the same as the length of shards in matrix | |
| num-shards: '8' | |
| # Validates source-built Spark for every PR using the SHA in this branch's metadata. | |
| source_spark_test: | |
| name: "DS Source Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}" | |
| runs-on: ubuntu-24.04 | |
| needs: [generate-matrix, prepare_spark_source_cache] | |
| # Run even if the non-blocking prepare job failed. If no cache was produced, each | |
| # shard fails during restore/verification under this already non-blocking job. | |
| if: ${{ always() && needs.generate-matrix.result == 'success' }} | |
| # Source-built Spark lanes track unreleased Spark APIs and remain non-blocking | |
| # while still producing a full pass/fail signal for every PR. | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| spark_version: ${{ fromJson(needs.generate-matrix.outputs.source_build_spark_versions) }} | |
| scala: [2.13.16] | |
| shard: [0, 1, 2, 3, 4, 5, 6, 7] | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: Prepare Spark environment (restore source cache) | |
| uses: ./.github/actions/prepare-spark-env | |
| with: | |
| spark-version: ${{ matrix.spark_version }} | |
| use-spark-source-cache: 'true' | |
| build-on-miss: 'false' | |
| cache-key-suffix: ${{ github.event.inputs['source-cache-key-suffix'] || '' }} | |
| - name: Run Delta Spark tests | |
| uses: ./.github/actions/run-delta-spark-tests | |
| with: | |
| spark-version: ${{ matrix.spark_version }} | |
| scala: ${{ matrix.scala }} | |
| shard: ${{ matrix.shard }} | |
| num-shards: '8' |