ggml-cuda: pad matmul weight rows to avoid cache-set aliasing

Weight matrices whose packed row size is a multiple of 2048 bytes map every row
onto the same L2 cache sets, so a matmul walking down a column thrashes a single
set. On RDNA3.5 this shows up as lost bandwidth in the FFN and attention GEMMs.

Add one cache line (128 B) to the row stride of such weights at buffer-init time
so consecutive rows land in different sets. get_alloc_size reserves the extra
bytes, and the loader uploads the packed file data with a strided 2D copy since
the destination rows are no longer adjacent. The gaps are zeroed so they cannot
hold NaNs.

Quantized weights are excluded: 128 is not a multiple of their block size and
would misalign the block-indexed kernels. The matmul paths need no change --
cuBLAS, mmf and mmvf all take the leading dimension from nb[1] already.

The loader flags the eligible tensors with GGML_TENSOR_FLAG_PAD_ROWS, reusing
llm_tensor_info_for() to tell a matmul weight from a bias or a norm; the
token-embedding/output duplication rule it shares with the buffer-type selection
is factored into resolve_tn_tensor() so the two cannot drift.

Gated to RDNA3.5; disable with GGML_CUDA_NO_PAD_WEIGHTS.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: roberteg16

ggml/include/ggml.h

@@ -648,11 +648,12 @@ extern "C" {
 
     // this tensor...
     enum ggml_tensor_flag {
-        GGML_TENSOR_FLAG_INPUT   =  1, // ...is an input for the GGML compute graph
-        GGML_TENSOR_FLAG_OUTPUT  =  2, // ...is an output for the GGML compute graph
-        GGML_TENSOR_FLAG_PARAM   =  4, // ...contains trainable parameters
-        GGML_TENSOR_FLAG_LOSS    =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)
-        GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed
+        GGML_TENSOR_FLAG_INPUT    =  1, // ...is an input for the GGML compute graph
+        GGML_TENSOR_FLAG_OUTPUT   =  2, // ...is an output for the GGML compute graph
+        GGML_TENSOR_FLAG_PARAM    =  4, // ...contains trainable parameters
+        GGML_TENSOR_FLAG_LOSS     =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)
+        GGML_TENSOR_FLAG_COMPUTE  = 16, // ...must be computed
+        GGML_TENSOR_FLAG_PAD_ROWS = 32, // ...is a matmul weight whose row stride a backend may pad to avoid cache-set aliasing
     };
 
     enum ggml_tri_type {

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -761,6 +761,33 @@ static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {
     return ctx->dev_ptr;
 }
 
+#define GGML_CUDA_ROW_ALIAS_STRIDE 2048 // rows of this size, or a multiple of it, map to the same cache sets
+#define GGML_CUDA_ROW_PAD           128 // one cache line, added to the row stride to break the aliasing
+
+// quantized weights are excluded: GGML_CUDA_ROW_PAD is not a multiple of their block size and would
+// misalign the block-indexed matmul kernels
+static bool ggml_cuda_should_pad_weight(const ggml_tensor * tensor, int device) {
+    static const bool padding_disabled = getenv("GGML_CUDA_NO_PAD_WEIGHTS") != nullptr;
+
+    if (padding_disabled || !(tensor->flags & GGML_TENSOR_FLAG_PAD_ROWS)) {
+        return false;
+    }
+
+    if (!GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[device].cc)) {
+        return false;
+    }
+
+    if (tensor->view_src != nullptr || ggml_is_quantized(tensor->type) || tensor->ne[1] <= 1) {
+        return false;
+    }
+
+    return ggml_row_size(tensor->type, tensor->ne[0]) % GGML_CUDA_ROW_ALIAS_STRIDE == 0;
+}
+
+static size_t ggml_cuda_padded_row_size(const ggml_tensor * tensor) {
+    return ggml_row_size(tensor->type, tensor->ne[0]) + GGML_CUDA_ROW_PAD;
+}
+
 static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
     ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
 
@@ -769,6 +796,17 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
         return GGML_STATUS_SUCCESS;
     }
 
+    if (ggml_cuda_should_pad_weight(tensor, ctx->device)) {
+        tensor->nb[1] = ggml_cuda_padded_row_size(tensor);
+        tensor->nb[2] = tensor->nb[1]*tensor->ne[1];
+        tensor->nb[3] = tensor->nb[2]*tensor->ne[2];
+
+        // the gaps between rows are never written, initialize them to 0 to avoid possible NaN values
+        ggml_cuda_set_device(ctx->device);
+        CUDA_CHECK(cudaMemset(tensor->data, 0, ggml_backend_buft_get_alloc_size(buffer->buft, tensor)));
+        return GGML_STATUS_SUCCESS;
+    }
+
     if (ggml_is_quantized(tensor->type) && tensor->view_src == nullptr && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
         // initialize padding to 0 to avoid possible NaN values
         const size_t original_size = ggml_nbytes(tensor);
@@ -929,6 +967,11 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t
         }
     }
 
+    // reserve room for the row stride that ggml_backend_cuda_buffer_init_tensor will assign
+    if (ggml_cuda_should_pad_weight(tensor, buft_ctx->device)) {
+        size = ggml_cuda_padded_row_size(tensor)*ggml_nrows(tensor);
+    }
+
     return size;
 }
 

src/llama-model-loader.cpp

@@ -1038,6 +1038,17 @@ static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hpara
     return nullptr;
 }
 
+// some models use the token embedding tensor as the output, but since these are used in different layers and with different ops
+// the tensor is duplicated
+// to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor
+static llm_tensor resolve_tn_tensor(const LLM_TN_IMPL & tn, int flags) {
+    if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & llama_model_loader::TENSOR_DUPLICATED)) {
+        return LLM_TENSOR_OUTPUT;
+    }
+
+    return tn.tensor;
+}
+
 struct ggml_tensor * llama_model_loader::create_tensor(
         const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,
         const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
@@ -1079,17 +1090,9 @@ struct ggml_tensor * llama_model_loader::create_tensor(
             throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str()));
         }
 
-        // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops
-        // the tensor is duplicated
-        // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor
-        llm_tensor tn_tensor = tn.tensor;
-        if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) {
-            tn_tensor = LLM_TENSOR_OUTPUT;
-        }
-
         llm_tensor_info info;
         try {
-            info = llm_tensor_info_for(tn_tensor);
+            info = llm_tensor_info_for(resolve_tn_tensor(tn, flags));
         } catch (const std::out_of_range & e) {
             throw std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str()));
         }
@@ -1271,6 +1274,16 @@ struct ggml_tensor * llama_model_loader::create_tensor(
     struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);
     ggml_set_name(tensor, ggml_get_name(cur));
 
+    // tensors with a "weight" suffix are used as the src0 of the op that they map to; flag the ones
+    // that feed a matrix multiplication so that backends may pad their row stride
+    const bool is_weight = tn.suffix == nullptr || strcmp(tn.suffix, "weight") == 0;
+    if (is_weight) {
+        const ggml_op op = llm_tensor_info_for(resolve_tn_tensor(tn, flags)).op;
+        if (op == GGML_OP_MUL_MAT || op == GGML_OP_MUL_MAT_ID) {
+            tensor->flags |= GGML_TENSOR_FLAG_PAD_ROWS;
+        }
+    }
+
     if (duplicated) {
         size_data += ggml_nbytes(cur);
     } else {
@@ -1527,7 +1540,13 @@ bool llama_model_loader::load_all_data(
             }
         }
 
-        size_t n_size = ggml_nbytes(cur);
+        // the data in the file is packed, while the destination tensor may have a padded row
+        // stride, in which case the rows are uploaded with a strided 2D copy
+        const size_t  packed_row_size = ggml_row_size(cur->type, cur->ne[0]);
+        const int64_t n_rows          = ggml_nelements(cur) / cur->ne[0];
+        const bool    row_padded      = cur->nb[1] != packed_row_size;
+
+        const size_t n_size = row_padded ? packed_row_size*n_rows : ggml_nbytes(cur);
 
         if (use_mmap) {
             const auto & mapping = mappings.at(weight->idx);
@@ -1554,6 +1573,8 @@ bool llama_model_loader::load_all_data(
                 auto & mmap_used = mmaps_used[weight->idx];
                 mmap_used.first  = std::min(mmap_used.first,  weight->offs);
                 mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);
+            } else if (row_padded) {
+                ggml_backend_tensor_set_2d(cur, data, 0, packed_row_size, n_rows, cur->nb[1], packed_row_size);
             } else {
                 ggml_backend_tensor_set(cur, data, 0, n_size);
             }
@@ -1570,7 +1591,7 @@ bool llama_model_loader::load_all_data(
                 }
             } else {
                 // If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.
-                if (upload_backend) {
+                if (upload_backend && !row_padded) {
                     size_t offset = weight->offs;
                     alignment = file->read_alignment();
                     size_t aligned_offset = offset & ~(alignment - 1);
@@ -1626,7 +1647,11 @@ bool llama_model_loader::load_all_data(
                     read_buf.resize(n_size);
                     file->seek(weight->offs, SEEK_SET);
                     file->read_raw(read_buf.data(), n_size);
-                    ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
+                    if (row_padded) {
+                        ggml_backend_tensor_set_2d(cur, read_buf.data(), 0, packed_row_size, n_rows, cur->nb[1], packed_row_size);
+                    } else {
+                        ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
+                    }
                     if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
                         throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
                     }