ggml-cuda: make the weight row padding per-architecture and tunable

The alias stride and the pad were compile-time constants gated to RDNA3.5. Both
follow from the cache geometry -- line size times number of sets -- so another
architecture needs different values, and neither CUDA nor HIP report enough to
derive them: hipDeviceProp_t::l2CacheSize is documented as always returning 0,
and set associativity is not exposed at all.

Move the pair into cuda_device_info, resolved once per device from a table keyed
on the compute capability, then overridden by GGML_CUDA_ROW_ALIAS_STRIDE and
GGML_CUDA_ROW_PAD. Porting to another architecture becomes a table entry, and
the values can be swept without rebuilding. Architectures with no entry keep
packed rows, so each one is opted in only after being measured.

Resolving once per device is also where the settings are validated: an alias
stride that is not a power of two, or a pad that is a multiple of it and so
leaves the rows aliasing, warns and falls back to no padding.
GGML_CUDA_NO_PAD_WEIGHTS now resolves to a zero pad, leaving a single way for
the padding to be off.

Add the per-type check that was missing: ggml_cuda_should_use_mmf and
ggml_cuda_should_use_mmvf reject a src0 whose strides are not a multiple of
2*type_size, so a pad that misses this would not fail or corrupt anything, it
would quietly stop those kernels from being selected.

Also correct the reason quantized weights are excluded. The matmul kernels do
read the row stride, as nb[1]/type_size in mmq.cu and mmvq.cu; what a 128 byte
pad breaks is that division being exact.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: roberteg16

ggml/src/ggml-cuda/common.cuh

@@ -1126,6 +1126,15 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> {
 
 //////////////////////
 
+// Row-stride padding for matrix multiplication weights. A weight whose packed row size is a
+// multiple of alias_stride puts every row in the same cache sets, so pad bytes are added to the
+// row stride to break the aliasing. Both values follow from the cache geometry (line size times
+// number of sets), which neither CUDA nor HIP report, so they are tabulated per architecture.
+struct ggml_cuda_row_pad_params {
+    size_t alias_stride; // packed row sizes that are a multiple of this alias in the cache
+    size_t pad;          // bytes added to the row stride; 0 disables the padding
+};
+
 struct ggml_cuda_device_info {
     int device_count;           // number of (possibly virtual) devices exposed to the rest of ggml
     int physical_device_count;  // number of physical CUDA devices actually present
@@ -1144,6 +1153,7 @@ struct ggml_cuda_device_info {
         int     physical_device;                // backing physical CUDA device for this (virtual) device
         int     physical_share_count;           // number of (virtual) devices sharing this device's physical GPU
         int     virtual_index;                  // index of this (virtual) device among those sharing its physical GPU
+        ggml_cuda_row_pad_params row_pad;       // weight row-stride padding
     };
 
     cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -220,6 +220,71 @@ static int ggml_cuda_parse_id(char devName[]) {
 }
 #endif // defined(GGML_USE_HIP)
 
+// Weight row-stride padding per architecture. An architecture is opted in only once the padding
+// has been measured on it; every other one gets {0, 0} and keeps packed rows.
+static ggml_cuda_row_pad_params ggml_cuda_row_pad_params_for(int cc) {
+    if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+        return { 2048, 128 }; // one cache line added to rows that alias every 2 KiB
+    }
+
+    return { 0, 0 };
+}
+
+// Returns the environment override for name, or -1 when it is unset or not a number.
+static int64_t ggml_cuda_row_pad_env(const char * name) {
+    const char * val = getenv(name);
+    if (!val || !val[0]) {
+        return -1;
+    }
+
+    char * end = nullptr;
+    const int64_t parsed = strtoll(val, &end, 0);
+    if (*end != '\0' || parsed < 0) {
+        GGML_LOG_WARN("%s: ignoring %s=%s, expected a non-negative integer\n", __func__, name, val);
+        return -1;
+    }
+
+    return parsed;
+}
+
+// Resolves the padding for one device: the architecture default, then the environment overrides,
+// then the sanity checks. Runs once per device so an invalid setting warns once, not per tensor.
+static ggml_cuda_row_pad_params ggml_cuda_resolve_row_pad(int cc) {
+    if (getenv("GGML_CUDA_NO_PAD_WEIGHTS")) {
+        return { 0, 0 };
+    }
+
+    ggml_cuda_row_pad_params params = ggml_cuda_row_pad_params_for(cc);
+
+    const int64_t alias_stride_override = ggml_cuda_row_pad_env("GGML_CUDA_ROW_ALIAS_STRIDE");
+    const int64_t pad_override          = ggml_cuda_row_pad_env("GGML_CUDA_ROW_PAD");
+    if (alias_stride_override >= 0) {
+        params.alias_stride = alias_stride_override;
+    }
+    if (pad_override >= 0) {
+        params.pad = pad_override;
+    }
+
+    if (params.pad == 0) {
+        return { 0, 0 };
+    }
+
+    if (params.alias_stride == 0 || (params.alias_stride & (params.alias_stride - 1)) != 0) {
+        GGML_LOG_WARN("%s: disabling weight row padding, alias stride %zu is not a power of two\n",
+                      __func__, params.alias_stride);
+        return { 0, 0 };
+    }
+
+    // a pad that is itself a multiple of the alias stride leaves the rows aliasing
+    if (params.pad % params.alias_stride == 0) {
+        GGML_LOG_WARN("%s: disabling weight row padding, pad %zu is a multiple of alias stride %zu\n",
+                      __func__, params.pad, params.alias_stride);
+        return { 0, 0 };
+    }
+
+    return params;
+}
+
 static ggml_cuda_device_info ggml_cuda_init() {
     ggml_cuda_device_info info = {};
 
@@ -376,6 +441,8 @@ static ggml_cuda_device_info ggml_cuda_init() {
         }
 
 #endif  // defined(GGML_USE_HIP)
+
+        info.devices[id].row_pad = ggml_cuda_resolve_row_pad(info.devices[id].cc);
     }
 
     if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {
@@ -761,31 +828,30 @@ static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {
     return ctx->dev_ptr;
 }
 
-#define GGML_CUDA_ROW_ALIAS_STRIDE 2048 // rows of this size, or a multiple of it, map to the same cache sets
-#define GGML_CUDA_ROW_PAD           128 // one cache line, added to the row stride to break the aliasing
-
-// quantized weights are excluded: GGML_CUDA_ROW_PAD is not a multiple of their block size and would
-// misalign the block-indexed matmul kernels
+// Quantized weights are excluded: the matmul kernels read the row stride as nb[1]/type_size, an
+// exact division that a pad which is not a multiple of the block size would break.
 static bool ggml_cuda_should_pad_weight(const ggml_tensor * tensor, int device) {
-    static const bool padding_disabled = getenv("GGML_CUDA_NO_PAD_WEIGHTS") != nullptr;
+    const ggml_cuda_row_pad_params & row_pad = ggml_cuda_info().devices[device].row_pad;
 
-    if (padding_disabled || !(tensor->flags & GGML_TENSOR_FLAG_PAD_ROWS)) {
+    if (row_pad.pad == 0 || !(tensor->flags & GGML_TENSOR_FLAG_PAD_ROWS)) {
         return false;
     }
 
-    if (!GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[device].cc)) {
+    if (tensor->view_src != nullptr || ggml_is_quantized(tensor->type) || tensor->ne[1] <= 1) {
         return false;
     }
 
-    if (tensor->view_src != nullptr || ggml_is_quantized(tensor->type) || tensor->ne[1] <= 1) {
+    // ggml_cuda_should_use_mmf and ggml_cuda_should_use_mmvf reject a src0 whose strides are not a
+    // multiple of 2*type_size, so a pad that misses this would silently cost those kernels
+    if (row_pad.pad % (2*ggml_type_size(tensor->type)) != 0) {
         return false;
     }
 
-    return ggml_row_size(tensor->type, tensor->ne[0]) % GGML_CUDA_ROW_ALIAS_STRIDE == 0;
+    return ggml_row_size(tensor->type, tensor->ne[0]) % row_pad.alias_stride == 0;
 }
 
-static size_t ggml_cuda_padded_row_size(const ggml_tensor * tensor) {
-    return ggml_row_size(tensor->type, tensor->ne[0]) + GGML_CUDA_ROW_PAD;
+static size_t ggml_cuda_padded_row_size(const ggml_tensor * tensor, int device) {
+    return ggml_row_size(tensor->type, tensor->ne[0]) + ggml_cuda_info().devices[device].row_pad.pad;
 }
 
 static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
@@ -797,7 +863,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
     }
 
     if (ggml_cuda_should_pad_weight(tensor, ctx->device)) {
-        tensor->nb[1] = ggml_cuda_padded_row_size(tensor);
+        tensor->nb[1] = ggml_cuda_padded_row_size(tensor, ctx->device);
         tensor->nb[2] = tensor->nb[1]*tensor->ne[1];
         tensor->nb[3] = tensor->nb[2]*tensor->ne[2];
 
@@ -969,7 +1035,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t
 
     // reserve room for the row stride that ggml_backend_cuda_buffer_init_tensor will assign
     if (ggml_cuda_should_pad_weight(tensor, buft_ctx->device)) {
-        size = ggml_cuda_padded_row_size(tensor)*ggml_nrows(tensor);
+        size = ggml_cuda_padded_row_size(tensor, buft_ctx->device)*ggml_nrows(tensor);
     }
 
     return size;