Arm backend: update Vulkan SDK setup for newer glslc (pytorch#20023)

update Vulkan SDK setup for newer glslc, which fixes testing time shader
compiles where the system had an old glslc (and revises the version used
in our test scripts)
Also mark custom shader tests as vgf so they run in just the VGF or
general testing, not in e.g. baremetal.


cc @SS-JIA @manuelcandales @digantdesai @cbilgin @freddan80 @per @zingo
@oscarandersson8218 @mansnils @Sebastian-Larsson @rascani

---------

Signed-off-by: Rob Elliott <Robert.Elliott@arm.com>

Author: robell

backends/arm/scripts/docgen/vgf/vgf-getting-started-tutorial.md.in

@@ -135,7 +135,7 @@ In this tutorial you have learned how to use ExecuTorch to export a PyTorch mode
 
 Issue: glslc is not found when configuring the executor runner.
 Solution: The Vulkan sdk is likely not in your path, check whether setup_path.sh contains something like
-`export PATH=$(pwd)/examples/arm/arm-scratch/vulkan_sdk/1.4.321.1/x86_64/bin:$PATH`.
+`export PATH=$(pwd)/examples/arm/arm-scratch/vulkan_sdk/1.4.341.1/x86_64/bin:$PATH`.
 If not, add it and source the file.
 
 If you encountered any bugs or issues following this tutorial please file a bug/issue here on [Github](https://github.com/pytorch/executorch/issues/new).

backends/arm/scripts/vulkan_utils.sh

@@ -17,28 +17,30 @@ fi
 script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
 source "${script_dir}/utils.sh"
 
-vulkan_sdk_version="1.4.321.1"
+vulkan_sdk_version=""
 vulkan_sdk_base_dir="vulkan_sdk"
 
 os_name="${OS:-$(uname -s)}"
 vulkan_sdk_arch="${ARCH}"
 
-# Vulkan SDK selection differs between macOS and Linux; macOS has its own SDK version
+# macOS and Linux x86_64 use the official LunarG SDK tarballs. Linux ARM64
+# uses a separately repackaged mirror of the same SDK version.
 if [[ "${os_name}" == "Darwin" ]]; then
-    # Latest published macOS SDK is 1.4.321.0 (1.4.321.1 is not available for macOS)
-    vulkan_sdk_version="1.4.321.0"
+    vulkan_sdk_version="1.4.341.1"
     vulkan_sdk_arch="macOS"
     vulkan_sdk_url="https://sdk.lunarg.com/sdk/download/${vulkan_sdk_version}/mac/vulkansdk-macos-${vulkan_sdk_version}.zip"
-    vulkan_sdk_sha256="d873c43acacec1e3330fb530dafd541aa5d8a5726575a98a3f70ca505fc203db"
+    vulkan_sdk_sha256="632cbe96c8ed6ed00c6ce25e3a7738c466134f76586e1c51f1419410d7f9042e"
 elif [[ "${os_name}" == "Linux" ]] && [[ "${ARCH}" == "x86_64" ]]; then
+    vulkan_sdk_version="1.4.341.1"
     vulkan_sdk_url="https://sdk.lunarg.com/sdk/download/${vulkan_sdk_version}/linux/vulkansdk-linux-x86_64-${vulkan_sdk_version}.tar.xz"
-    vulkan_sdk_sha256="f22a3625bd4d7a32e7a0d926ace16d5278c149e938dac63cecc00537626cbf73"
+    vulkan_sdk_sha256="3bf0f762afb6c79bc6a9d9fb5998745ccff928800a29619b501ed9de7fd9789b"
 elif [[ "${os_name}" == "Linux" ]] && ([[ "${ARCH}" == "aarch64" ]] || [[ "${ARCH}" == "arm64" ]]); then
+    vulkan_sdk_version="1.4.341.1"
     if [[ "${vulkan_sdk_arch}" == "arm64" ]]; then
         vulkan_sdk_arch="aarch64"
     fi
-    vulkan_sdk_url="https://github.com/jakoch/vulkan-sdk-arm/releases/download/1.4.321.1/vulkansdk-ubuntu-22.04-arm-1.4.321.1.tar.xz"
-    vulkan_sdk_sha256="c57e318d0940394d3a304034bb7ddabda788b5b0b54638e80e90f7264efe9f84"
+    vulkan_sdk_url="https://github.com/jakoch/vulkan-sdk-arm/releases/download/${vulkan_sdk_version}/vulkansdk-ubuntu-22.04-arm-${vulkan_sdk_version}.tar.xz"
+    vulkan_sdk_sha256="345312aee2c835e128b30653278593f899a659a7ba287c571cafb22acb708b8f"
 else
     log_step "vulkan" "Error: only macOS and Linux are supported (detected ${os_name}); architecture must be x86-64 or aarch64/arm64"
     exit 1
@@ -164,7 +166,8 @@ function setup_path_vulkan() {
     vulkan_sdk_arch_root="$(cd "${vulkan_sdk_arch_root}" && pwd)"
     vulkan_sdk_bin_path="$(cd "${vulkan_sdk_bin_dir}" && pwd)"
 
-    append_env_in_setup_path PATH "${vulkan_sdk_bin_path}"
+    # Prefer the SDK-provided compiler over any host-installed glslc.
+    prepend_env_in_setup_path PATH "${vulkan_sdk_bin_path}"
     if [[ "${OS:-}" == "Darwin" ]]; then
         prepend_env_in_setup_path DYLD_LIBRARY_PATH "${vulkan_sdk_arch_root}/lib"
         local moltenvk_icd_path="${vulkan_sdk_arch_root}/share/vulkan/icd.d/MoltenVK_icd.json"

backends/arm/test/misc/test_custom_shader_payloads.py

@@ -100,7 +100,7 @@ def _decode_sampler_payload(
 
 # Covers basic payload encoding and decoding for shader metadata.
 # Checks bindings, workgroup sizes, language, and formats are preserved.
-def test_buffer_shader_payload_encodes_bindings_and_formats():
+def test_buffer_shader_payload_vgf_encodes_bindings_and_formats():
     payload = decode_payload(
         encode_payload(
             build_grid_sampler_2d_payload(
@@ -124,7 +124,7 @@ def test_buffer_shader_payload_encodes_bindings_and_formats():
 
 # Covers sampler-specific payload fields for sampled image inputs.
 # Checks filter, address mode, and border color are encoded in the payload.
-def test_sampler_shader_payload_encodes_sampler_fields():
+def test_sampler_shader_payload_vgf_encodes_sampler_fields():
     payload = _decode_sampler_payload()
 
     assert (
@@ -145,7 +145,7 @@ def test_sampler_shader_payload_encodes_sampler_fields():
 
 # Covers the local shader asset contract used by the tests.
 # Checks the expected GLSL/SPIR-V asset names and that the SPIR-V bytes look valid.
-def test_shader_payload_uses_expected_glsl_and_spirv_asset():
+def test_shader_payload_vgf_uses_expected_glsl_and_spirv_asset():
     buffer_payload = build_grid_sampler_2d_payload(
         interpolation_mode=0,
         padding_mode=0,
@@ -160,7 +160,7 @@ def test_shader_payload_uses_expected_glsl_and_spirv_asset():
 
 # Covers validation of unsupported shader option values.
 # Checks invalid mode and padding_mode values raise instead of encoding silently.
-def test_shader_payload_rejects_invalid_mode_values():
+def test_shader_payload_vgf_rejects_invalid_mode_values():
     with pytest.raises(RuntimeError, match="Unsupported grid_sample mode"):
         _decode_sampler_payload(mode="garbage")
 
@@ -170,7 +170,7 @@ def test_shader_payload_rejects_invalid_mode_values():
 
 # Covers storage-image outputs, which should not carry sampler state.
 # Checks output payloads omit sampler metadata for storage images.
-def test_storage_image_payload_does_not_require_sampler_fields():
+def test_storage_image_payload_vgf_does_not_require_sampler_fields():
     payload = _decode_sampler_payload()
 
     assert payload["output_0_vkdescriptortype"] == "VK_DESCRIPTOR_TYPE_STORAGE_IMAGE"

backends/arm/test/ops/test_custom_shader_lowering.py

@@ -79,7 +79,7 @@ def forward(self, x: torch.Tensor, grid: torch.Tensor) -> torch.Tensor:
 
 # Covers lowering of a standalone custom op to a buffer-backed tosa.CUSTOM.
 # Checks the emitted custom node carries the expected operator, domain, and buffer descriptors.
-def test_new_custom_op_lowers_to_tosa_custom_buffer_shader():
+def test_new_custom_op_vgf_lowers_to_tosa_custom_buffer_shader():
     if shutil.which("glslc") is None:
         pytest.skip("glslc not found")
     register_test_threes_library_ops()
@@ -105,7 +105,7 @@ def test_new_custom_op_lowers_to_tosa_custom_buffer_shader():
 
 # Covers replacing aten.add with a shader-backed custom op.
 # Checks the rewritten node lowers to tosa.CUSTOM with storage-buffer descriptors.
-def test_replacement_op_lowers_to_tosa_custom_shader():
+def test_replacement_op_vgf_lowers_to_tosa_custom_shader():
     if shutil.which("glslc") is None:
         pytest.skip("glslc not found")
     register_test_shader_library_ops()
@@ -132,7 +132,7 @@ def test_replacement_op_lowers_to_tosa_custom_shader():
 
 # Covers the in-tree grid-sampler rewrite path.
 # Checks grid_sampler_2d.default lowers to tosa.CUSTOM with the Vulkan shader domain.
-def test_in_tree_grid_sampler_lowers_to_tosa_custom():
+def test_in_tree_grid_sampler_vgf_lowers_to_tosa_custom():
     edge_model = to_edge(
         export(_GridSampleModule(), (torch.randn(1, 3, 8, 8), torch.randn(1, 4, 4, 2)))
     )
@@ -155,7 +155,7 @@ def test_in_tree_grid_sampler_lowers_to_tosa_custom():
 
 # Covers sampler/image descriptor selection during lowering.
 # Checks the lowered payload uses combined-image-sampler input, tensor grid input, and storage-image output.
-def test_sampler_shader_lowering_emits_expected_descriptor_types():
+def test_sampler_shader_vgf_lowering_emits_expected_descriptor_types():
     if shutil.which("glslc") is None:
         pytest.skip("glslc not found")
     register_test_shader_library_ops()
@@ -188,7 +188,7 @@ def test_sampler_shader_lowering_emits_expected_descriptor_types():
     assert payload["output_0_vkdescriptortype"] == "VK_DESCRIPTOR_TYPE_STORAGE_IMAGE"
 
 
-def test_grid_read_shader_lowering_uses_distinct_custom_operator():
+def test_grid_read_shader_vgf_lowering_uses_distinct_custom_operator():
     if shutil.which("glslc") is None:
         pytest.skip("glslc not found")
     register_test_shader_library_ops()
@@ -212,7 +212,7 @@ def test_grid_read_shader_lowering_uses_distinct_custom_operator():
     assert custom_node.kwargs["operator_name"] == TEST_GRID_READ_TENSOR_OPERATOR
 
 
-def test_sampler_shader_lowering_rejects_three_channel_image_payload():
+def test_sampler_shader_vgf_lowering_rejects_three_channel_image_payload():
     if shutil.which("glslc") is None:
         pytest.skip("glslc not found")
     register_test_shader_library_ops()
@@ -237,7 +237,7 @@ def test_sampler_shader_lowering_rejects_three_channel_image_payload():
 
 # Covers decoding of implementation_attrs after lowering.
 # Checks the payload exposes the expected entry point and binding numbering.
-def test_shader_lowering_decodes_expected_implementation_attrs():
+def test_shader_lowering_vgf_decodes_expected_implementation_attrs():
     edge_model = to_edge(
         export(_GridSampleModule(), (torch.randn(1, 3, 8, 8), torch.randn(1, 4, 4, 2)))
     )

backends/arm/test/passes/test_rewrite_grid_sampler_to_tosa_custom_pass.py

@@ -44,7 +44,7 @@ def forward(self, x, grid):
         )
 
 
-def test_rewrite_grid_sampler_to_tosa_custom_no_target():
+def test_rewrite_grid_sampler_to_tosa_custom_vgf_no_target():
     model = GridSampler2d()
     example_inputs = (
         torch.randn(1, 3, 8, 8),

backends/vulkan/cmake/ShaderLibrary.cmake

@@ -30,7 +30,7 @@ if(NOT GLSLC_PATH AND EXECUTORCH_BUILD_VULKAN)
   message(
     FATAL_ERROR
       "glslc from the Vulkan SDK must be installed to build the Vulkan backend. "
-      "Please install the Vulkan SDK 1.4.321.0 or newer from "
+      "Please install the Vulkan SDK 1.4.341.1 or newer from "
       "https://vulkan.lunarg.com/sdk/home and ensure that the glslc binary is in your PATH. "
       "Note that the glslc distributed with the Android NDK is not compatible since it "
       "does not support the GL_EXT_integer_dot_product extension. "

docs/source/backends/arm-vgf/tutorials/vgf-getting-started.md

@@ -219,7 +219,7 @@ In this tutorial you have learned how to use ExecuTorch to export a PyTorch mode
 
 Issue: glslc is not found when configuring the executor runner.
 Solution: The Vulkan sdk is likely not in your path, check whether setup_path.sh contains something like
-`export PATH=$(pwd)/examples/arm/arm-scratch/vulkan_sdk/1.4.321.1/x86_64/bin:$PATH`.
+`export PATH=$(pwd)/examples/arm/arm-scratch/vulkan_sdk/1.4.341.1/x86_64/bin:$PATH`.
 If not, add it and source the file.
 
 If you encountered any bugs or issues following this tutorial please file a bug/issue here on [Github](https://github.com/pytorch/executorch/issues/new).