|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#version 450 core |
| 10 | + |
| 11 | +${define_required_extensions(STORAGE, DTYPE)} |
| 12 | +${define_required_extensions("buffer", DTYPE)} |
| 13 | + |
| 14 | +#define PRECISION ${PRECISION} |
| 15 | + |
| 16 | +#define VEC4_T ${texel_load_type(DTYPE, STORAGE)} |
| 17 | +#define BUFFER_VEC4_T ${texel_load_type(DTYPE, "buffer")} |
| 18 | + |
| 19 | +${define_active_storage_type(STORAGE)} |
| 20 | + |
| 21 | +layout(std430) buffer; |
| 22 | + |
| 23 | +#include "indexing.glslh" |
| 24 | + |
| 25 | +${layout_declare_tensor(B, "w", "t_out", DTYPE, STORAGE, is_scalar_array=False)} |
| 26 | +${layout_declare_tensor(B, "r", "t_in", DTYPE, STORAGE, is_scalar_array=False)} |
| 27 | +// freqs_cis is always bound as a buffer so the shader can flat-index it |
| 28 | +// regardless of the caller's declared rank (2D [N, C] or 4D [1, N, C/2, 2]). |
| 29 | +${layout_declare_tensor(B, "r", "t_freqs", DTYPE, "buffer", is_scalar_array=False)} |
| 30 | + |
| 31 | +$if STORAGE == "buffer": |
| 32 | + ${layout_declare_ubo(B, "BufferMetadata", "outp")} |
| 33 | +$else: |
| 34 | + ${layout_declare_ubo(B, "TextureMetadata", "outp")} |
| 35 | + |
| 36 | +layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; |
| 37 | + |
| 38 | +${layout_declare_spec_const(C, "int", "outp_layout", "CONTIG_LAYOUT_INT")} |
| 39 | + |
| 40 | +/* |
| 41 | + * Applies rotary positional embeddings to a tensor whose last dimension |
| 42 | + * contains pair-interleaved (real, imag) components. This matches EdgeTAM's |
| 43 | + * `apply_rotary_enc_without_complex` semantics, where the fused cos/sin |
| 44 | + * freqs tensor has a flattened [N, C] layout (cos, sin pairs interleaved). |
| 45 | + * |
| 46 | + * Inputs: |
| 47 | + * t_in: [B, N, C] (last dim packed as [r0, i0, r1, i1, ...]) |
| 48 | + * t_freqs: contiguous memory with N * C elements. May arrive at any rank |
| 49 | + * (e.g. 2D [N, C] or 4D [1, N, C/2, 2]). Physically the values |
| 50 | + * are [cos0, sin0, cos1, sin1, ...] along the final axis. |
| 51 | + * |
| 52 | + * Output: |
| 53 | + * t_out: same shape as t_in |
| 54 | + * |
| 55 | + * Math per k in [0, C/2): |
| 56 | + * out[2k] = x[2k] * cos[k] - x[2k+1] * sin[k] |
| 57 | + * out[2k+1] = x[2k] * sin[k] + x[2k+1] * cos[k] |
| 58 | + * |
| 59 | + * Each thread processes one width-packed texel (4 elements = 2 (r, i) pairs). |
| 60 | + * All participating tensors are assumed to be width-packed with standard axis |
| 61 | + * maps. |
| 62 | + * |
| 63 | + * The freqs tensor is indexed using a flat (n_idx * C + c_offset) address to |
| 64 | + * remain correct regardless of input rank — the shape of t_freqs does not |
| 65 | + * need to match the logical [N, C] layout, only the underlying memory does. |
| 66 | + */ |
| 67 | +void main() { |
| 68 | + // Each thread computes one output texel of 4 elements along the last dim. |
| 69 | + TensorIndex4D out_tidx = zero_tensor4d_idx(); |
| 70 | + out_tidx.data.x = int(gl_GlobalInvocationID.x) * 4; |
| 71 | + out_tidx.data.y = int(gl_GlobalInvocationID.y); |
| 72 | + out_tidx.data.z = int(gl_GlobalInvocationID.z); |
| 73 | + |
| 74 | + if (out_of_bounds(out_tidx, outp)) { |
| 75 | + return; |
| 76 | + } |
| 77 | + |
| 78 | + // Freqs tensor is always a contiguous buffer of N * C elements. Compute |
| 79 | + // a flat texel index directly from logical (n_idx, c_elem_idx / 4). The |
| 80 | + // logical width C comes from the output tensor metadata — both buffer |
| 81 | + // and texture metadata store this at index 0 (sizes[0][0] / sizes.x). |
| 82 | +#ifdef USING_BUFFER |
| 83 | + const uint C_width = outp.sizes[0][0]; |
| 84 | +#else |
| 85 | + const uint C_width = uint(outp.sizes.x); |
| 86 | +#endif |
| 87 | + const uint freqs_texel_bufi = |
| 88 | + uint(out_tidx.data.y) * div_4(C_width) |
| 89 | + + uint(gl_GlobalInvocationID.x); |
| 90 | + BUFFER_VEC4_T f_tex = t_freqs[freqs_texel_bufi]; |
| 91 | + |
| 92 | +#ifdef USING_BUFFER |
| 93 | + const uint out_texel_bufi = |
| 94 | + div_4(tensor4d_idx_to_linear_idx(outp, out_tidx)); |
| 95 | + VEC4_T x_tex = t_in[out_texel_bufi]; |
| 96 | +#else // USING_TEXTURE |
| 97 | + const ivec3 out_pos = |
| 98 | + tensor4d_idx_to_texel_pos_simple(outp, out_tidx, outp_layout); |
| 99 | + VEC4_T x_tex = texelFetch(t_in, out_pos, 0); |
| 100 | +#endif |
| 101 | + |
| 102 | + // x_tex = (r0, i0, r1, i1), f_tex = (c0, s0, c1, s1) |
| 103 | + VEC4_T out_tex; |
| 104 | + out_tex.x = x_tex.x * VEC4_T(f_tex).x - x_tex.y * VEC4_T(f_tex).y; |
| 105 | + out_tex.y = x_tex.x * VEC4_T(f_tex).y + x_tex.y * VEC4_T(f_tex).x; |
| 106 | + out_tex.z = x_tex.z * VEC4_T(f_tex).z - x_tex.w * VEC4_T(f_tex).w; |
| 107 | + out_tex.w = x_tex.z * VEC4_T(f_tex).w + x_tex.w * VEC4_T(f_tex).z; |
| 108 | + |
| 109 | +#ifdef USING_BUFFER |
| 110 | + t_out[out_texel_bufi] = out_tex; |
| 111 | +#else |
| 112 | + imageStore(t_out, out_pos, out_tex); |
| 113 | +#endif |
| 114 | +} |
0 commit comments