Skip to content

Commit dd7464a

Browse files
authored
Update lora def (pytorch#18342)
### Summary Update lora def to use nn.linear instead of torch.nn.functional.linear 1. Remove debug string from FQN. Otherwise we get different quant indices for lora and non-lora (because lora_a is quantized), after this change, and the foundation weights are no longer shareable. 2. Having nn.Linear as a submodule allows torchao quant to capture it, and we don't need custom logic filtering LoraLinear. as lora_a and lora_b weren't captured before, exclude these to maintain current behavior. Later we can remove this, optionally. 3. Having nn.Linear as submodule also means we need to remap lora weights, as weight names have an extra 'linear' in them. 4. Add @Property for weight, bias so that it remains BC and can be treated as a regular linear module. This is used in [load_weights_from_attention.py](https://github.com/pytorch/executorch/blob/eb92ceca74d0bace6d01115ae7f860b3baed5c2c/examples/models/llama/static_attention.py#L1154 ### Test plan CI
1 parent fd1581d commit dd7464a

3 files changed

Lines changed: 28 additions & 35 deletions

File tree

‎backends/xnnpack/operators/node_visitor.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,7 @@ def get_serialized_buffer_index(
625625
f"Serializing constant data node {tensor} but tensor value has no bytes",
626626
)
627627
sha256_hash = hashlib.sha256(bytes(array))
628-
named_key = tensor.name + "_" + sha256_hash.hexdigest()
628+
named_key = sha256_hash.hexdigest()
629629

630630
size = const_val.untyped_storage().nbytes()
631631
xnn_graph.constant_data.append(

‎examples/models/llama/lora.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,31 @@ def __init__(
2626
self.rank = rank
2727
self.alpha = alpha
2828
self.use_bias = use_bias
29-
self.dropout = dropout
30-
31-
linear = nn.Linear(in_dim, out_dim, bias=use_bias)
32-
weight = linear.weight
33-
bias = linear.bias if self.use_bias else None
34-
self.register_parameter("weight", nn.Parameter(weight))
35-
self.register_parameter(
36-
"bias", nn.Parameter(bias) if bias is not None else None
37-
)
3829

30+
self.linear = nn.Linear(in_dim, out_dim, bias=use_bias)
3931
self.dropout = nn.Dropout(p=dropout) if dropout > 0.0 else nn.Identity()
4032
self.lora_a = nn.Linear(in_features=in_dim, out_features=rank, bias=False)
4133
self.lora_b = nn.Linear(in_features=rank, out_features=out_dim, bias=False)
4234

35+
@property
36+
def weight(self):
37+
return self.linear.weight
38+
39+
@property
40+
def bias(self):
41+
return self.linear.bias
42+
43+
def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
44+
# Remap keys to "linear.*"
45+
for attr in ("weight", "bias"):
46+
old_key = prefix + attr
47+
new_key = prefix + "linear." + attr
48+
if old_key in state_dict and new_key not in state_dict:
49+
state_dict[new_key] = state_dict.pop(old_key)
50+
super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
51+
4352
def forward(self, x: torch.Tensor) -> torch.Tensor:
44-
out = torch.nn.functional.linear(x, self.weight, self.bias)
53+
out = self.linear(x)
4554
lora_out = self.lora_a(self.dropout(x))
4655
lora_out = (self.alpha / self.rank) * self.lora_b(lora_out)
4756

‎examples/models/llama/source_transformation/quantize.py‎

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -144,30 +144,14 @@ def quantize( # noqa C901
144144
from torchao.utils import unwrap_tensor_subclass
145145

146146
def filter_fn(m, fqn):
147-
# Check if it's a regular nn.Linear
148-
is_linear = isinstance(m, nn.Linear)
149-
150-
# Check if it's a LoRALinear (which has a base weight parameter to quantize)
151-
is_lora_linear = False
152-
try:
153-
from executorch.examples.models.llama.lora import LoRALinear
154-
155-
is_lora_linear = isinstance(m, LoRALinear)
156-
except ImportError:
157-
pass
158-
159-
# Check if the weight shape is compatible with group size
160-
has_shape_compatible_with_group_size = False
161-
if is_linear or is_lora_linear:
162-
if group_size == 0:
163-
has_shape_compatible_with_group_size = True
164-
else:
165-
has_shape_compatible_with_group_size = (
166-
m.weight.shape[1] % group_size == 0
167-
)
168-
return (
169-
is_linear or is_lora_linear
170-
) and has_shape_compatible_with_group_size
147+
if not isinstance(m, nn.Linear):
148+
return False
149+
parts = fqn.split(".")
150+
if "lora_a" in parts or "lora_b" in parts:
151+
return False
152+
if group_size == 0:
153+
return True
154+
return m.weight.shape[1] % group_size == 0
171155

172156
weight_dtype = torch.int4 if qmode == "8da4w" else torch.int8
173157
quantize_(

0 commit comments

Comments
 (0)