Add hybrid DLS cross-plugin integration tests (#1957) #7290
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test Neural Search | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every night | |
| push: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| pull_request: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| Get-CI-Image-Tag: | |
| uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main | |
| with: | |
| product: opensearch | |
| # The JNI layer is C++ and has its own googletest suite under jni/tests. The Java | |
| # `test` task builds the shared library but never runs these, so without a | |
| # dedicated job the native code ships untested. | |
| # | |
| # In the container, not on a bare runner: ubuntu-latest would test against a GCC | |
| # and an OpenMP runtime (LLVM libomp, via libomp-dev) that never ship. | |
| # arm64 is the only config where SVE_ENABLED holds, so the only one that builds | |
| # nsparse_sve. | |
| JNI-native-tests-linux: | |
| needs: Get-CI-Image-Tag | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, ubuntu-24.04-arm] | |
| name: JNI C++ Tests (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| # Same image opensearch-build uses for the distribution. Multi-arch manifest, | |
| # so this tag resolves to the arm64 image on an arm64 runner. | |
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | |
| # need to switch to root so that github actions can install runner binary on container without permission issues. | |
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | |
| steps: | |
| - name: Run start commands | |
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| java-version: 21 | |
| distribution: temurin | |
| cache: 'gradle' | |
| # No toolchain step needed: the image carries cmake, gcc-toolset-13 and libgomp. | |
| # su to 1000 because gradle refuses to run as root. | |
| - name: Run JNI C++ tests | |
| run: | | |
| chown -R 1000:1000 `pwd` | |
| su `id -un 1000` -c "./gradlew jniTest" | |
| # No CI-runner container for Windows. init-nsparse.cmake pins it to generic | |
| # nsparse, so this covers the MSVC toolchain and packaging path, not SIMD. | |
| JNI-native-tests-windows: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Windows builds with MSVC via the Visual Studio generator; the runner image | |
| # already ships VS 2022 and CMake, so no toolchain step is needed there. | |
| os: [windows-latest] | |
| name: JNI C++ Tests (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| java-version: 21 | |
| distribution: temurin | |
| cache: 'gradle' | |
| - name: Run JNI C++ tests | |
| run: ./gradlew jniTest | |
| # Leaks and use-after-free across the JNI boundary are exactly what assertions | |
| # cannot see: the layer hands raw pointers to Java and frees them on the native | |
| # side. This job is the real coverage for the ownership rules. Linux only: the | |
| # -fsanitize flags are GCC/Clang, and MSVC's /fsanitize=address has no LSan. | |
| # | |
| # Stays on a bare runner unlike JNI-native-tests-linux: the CI image's | |
| # gcc-toolset-13 has no libasan, so -fsanitize=address cannot link there. | |
| JNI-native-tests-sanitizers: | |
| name: JNI C++ Tests (ASan/LSan) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| java-version: 21 | |
| distribution: temurin | |
| cache: 'gradle' | |
| - name: Install native toolchain | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y cmake build-essential libomp-dev | |
| - name: Run JNI C++ tests under AddressSanitizer and LeakSanitizer | |
| run: ./gradlew jniTest -PjniSanitizers=true | |
| Check-neural-search-linux: | |
| needs: Get-CI-Image-Tag | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [21, 25] | |
| os: [ubuntu-latest] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| name: Gradle Check Linux | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| # using the same image which is used by opensearch-build team to build the OpenSearch Distribution | |
| # this image tag is subject to change as more dependencies and updates will arrive over time | |
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | |
| # need to switch to root so that github actions can install runner binary on container without permission issues. | |
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | |
| steps: | |
| - name: Run start commands | |
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| cache: 'gradle' | |
| - name: Run build | |
| run: | | |
| chown -R 1000:1000 `pwd` | |
| su `id -un 1000` -c "./gradlew check --parallel -x integTest" | |
| - name: Upload Coverage Report | |
| if: ${{ !cancelled() && contains(matrix.os, 'ubuntu') && contains(matrix.java, '21') }} | |
| uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| use_oidc: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL != '' }} | |
| use_pypi: true | |
| Check-neural-search-windows: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [21, 25] | |
| os: [windows-latest] | |
| name: Gradle Check Windows | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| cache: 'gradle' | |
| - name: Run build | |
| run: | | |
| ./gradlew check --parallel -x integTest | |
| Precommit-neural-search-linux: | |
| needs: Get-CI-Image-Tag | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [21, 25] | |
| os: [ubuntu-latest] | |
| name: Pre-commit Linux | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| # using the same image which is used by opensearch-build team to build the OpenSearch Distribution | |
| # this image tag is subject to change as more dependencies and updates will arrive over time | |
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | |
| # need to switch to root so that github actions can install runner binary on container without permission issues. | |
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | |
| steps: | |
| - name: Run start commands | |
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| cache: 'gradle' | |
| - name: Run build | |
| run: | | |
| chown -R 1000:1000 `pwd` | |
| su `id -un 1000` -c "./gradlew precommit -x integTest --parallel" | |
| integTest: | |
| needs: Precommit-neural-search-linux | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 21, 25 ] | |
| os: [ ubuntu-latest ] | |
| name: Integ Test JDK${{ matrix.java }}, ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| cache: 'gradle' | |
| distribution: temurin | |
| - name: Build and Run Tests | |
| run: | | |
| ./gradlew integTest --parallel | |
| integMultiNodeTest: | |
| needs: Precommit-neural-search-linux | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 21 ] | |
| os: [ ubuntu-latest ] | |
| configureNodeRoles: [ false, true ] | |
| name: Multi-Node Integ Test JDK${{ matrix.java }}, With Role Assignment ${{ matrix.configureNodeRoles }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # jni/external/neural-sparse-cpp is a submodule the native build needs | |
| submodules: recursive | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: temurin | |
| cache: 'gradle' | |
| - name: Build and Run Tests | |
| run: | | |
| ./gradlew integTest --parallel -PnumNodes=3 -PconfigureNodeRoles=${{ matrix.configureNodeRoles }} |