Skip to content

Commit b6aed42

Browse files
Qualcomm AI Engine Direct - Move encoder metadata into encoder classes (pytorch#22828)
### Summary: - Add get_metadata methods to audio and vision encoders. - Use encoder-owned metadata for layer sharding and quantize IO tagging. - add sharding disable log message ### Test plan LLM CI
1 parent 9f209bd commit b6aed42

3 files changed

Lines changed: 62 additions & 23 deletions

File tree

‎examples/qualcomm/oss_scripts/llama/model/audio_encoder.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,12 @@ def get_example_inputs(self):
151151
input_dim = self.config.encoder_config.input_dim
152152
return (torch.randn((1, self.n_bins, input_dim), dtype=torch.float32),)
153153

154+
def get_metadata(self):
155+
return {
156+
"get_context_size": self.config.encoder_config.context_size,
157+
"get_n_layers": self.config.encoder_config.num_layers,
158+
}
159+
154160
def forward(self, hidden_states: torch.Tensor):
155161
encoder_embeds = self.encoder(hidden_states)
156162
projected_embeds = self.projector(encoder_embeds)

‎examples/qualcomm/oss_scripts/llama/model/vision_encoder.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,11 @@ def get_example_inputs(self):
183183
),
184184
)
185185

186+
def get_metadata(self):
187+
return {
188+
"get_n_layers": self.config.vision_config.num_hidden_layers,
189+
}
190+
186191
def forward(
187192
self,
188193
pixel_values: torch.FloatTensor,
@@ -258,6 +263,11 @@ def get_example_inputs(self):
258263
),
259264
)
260265

266+
def get_metadata(self):
267+
return {
268+
"get_n_layers": self.config.vision_config.num_hidden_layers,
269+
}
270+
261271
def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5):
262272
"""Perform pixel shuffle downsampling on vision features.
263273

‎examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py‎

Lines changed: 46 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -223,23 +223,32 @@ def __init__(
223223
self.tok_embedding, self.decoder = self._prepare_model()
224224

225225
# check if sharding required
226-
if self.decoder and self.config.num_sharding > 1:
227-
layer_prefix_offsets = None
228-
if self.control_args.decoder_model == "gemma4-e2b":
229-
n_self = self.gemma4_config.num_self_decoder_layers
230-
layer_prefix_offsets = {
231-
"model.self_decoder.layers": 0,
232-
"model.cross_decoder.layers": n_self,
233-
}
234-
SplitGraph, setting = model_sharding.get_split_graph_pass(
235-
self.meta["get_n_layers"],
236-
shares=self.config.num_sharding,
237-
pattern=self._get_sharding_get_pattern(),
238-
layer_prefix_offsets=layer_prefix_offsets,
239-
)
240-
self.passes_job[SplitGraph] = setting
241-
self.dep_table[SplitGraph] = [FoldQDQ]
242-
self.dep_table[TagQuantIO] = [SplitGraph]
226+
if self.decoder is not None:
227+
if self.meta["get_n_layers"] >= self.config.num_sharding > 1:
228+
layer_prefix_offsets = None
229+
if self.control_args.decoder_model == "gemma4-e2b":
230+
n_self = self.gemma4_config.num_self_decoder_layers
231+
layer_prefix_offsets = {
232+
"model.self_decoder.layers": 0,
233+
"model.cross_decoder.layers": n_self,
234+
}
235+
SplitGraph, setting = model_sharding.get_split_graph_pass(
236+
self.meta["get_n_layers"],
237+
shares=self.config.num_sharding,
238+
pattern=self._get_sharding_get_pattern(),
239+
layer_prefix_offsets=layer_prefix_offsets,
240+
)
241+
self.passes_job[SplitGraph] = setting
242+
self.dep_table[SplitGraph] = [FoldQDQ]
243+
self.dep_table[TagQuantIO] = [SplitGraph]
244+
else:
245+
logging.info(
246+
f"Disabling sharding because the requested number of shards "
247+
f"({self.config.num_sharding}) is not valid for the number of layers "
248+
f"({self.meta['get_n_layers']}). "
249+
"Sharding is enabled only when the number of shards is greater than 1 "
250+
"and no greater than the number of layers."
251+
)
243252

244253
self._decoder_inference = (
245254
DecoderInference(
@@ -1253,20 +1262,23 @@ def __init__(
12531262
if config := getattr(config, modality, None):
12541263
if modality == AUDIO_ENCODER:
12551264
auto_model = AutoModelForSpeechSeq2Seq.from_pretrained(repo_id)
1256-
self.num_layers = auto_model.config.encoder_config.num_layers
1257-
self.ctx_size = auto_model.config.encoder_config.context_size
12581265
elif modality == TEXT_ENCODER:
12591266
raise NotImplementedError(f"{modality} is under development")
12601267
elif modality == VISION_ENCODER:
12611268
auto_model = AutoModel.from_pretrained(
12621269
repo_id, _attn_implementation="eager"
12631270
)
1264-
self.num_layers = auto_model.config.vision_config.num_hidden_layers
12651271
else:
12661272
raise NotImplementedError(f"Find no {modality}")
12671273

12681274
auto_model = auto_model.to(torch.float32).eval()
12691275
self.model = config().create_encoder(auto_model.config).eval()
1276+
self.meta = self.model.get_metadata()
1277+
if "get_n_layers" not in self.meta:
1278+
raise ValueError(
1279+
f"{type(self.model).__name__}.get_metadata() must provide "
1280+
"'get_n_layers'"
1281+
)
12701282
self.model.load_state_dict(
12711283
auto_model.state_dict(), strict=False
12721284
) # set strict to false to simplify parameter loading for non-text models
@@ -1298,12 +1310,15 @@ def _tag_ios(self, node, fixed_point_type):
12981310
quant_io_type = fixed_point_type["io_type"]
12991311

13001312
# GraniteSpeech: tag _to_copy op as quantized tensors for attn dist. It is caused by sharding
1313+
ctx_size = self.meta.get("get_context_size")
13011314
if (
13021315
issubclass(self.config, GraniteSpeechEncoder)
13031316
and node.target == exir_ops.edge.aten._to_copy.default
1304-
and node.meta["val"].size() == (self.ctx_size, self.ctx_size)
1317+
and ctx_size is not None
1318+
and node.meta["val"].size() == (ctx_size, ctx_size)
13051319
):
13061320
quant_io_type = torch.int32
1321+
logging.info(f"Tagging {node} with {quant_io_type} quantized IO ")
13071322

13081323
return quant_io_type
13091324

@@ -1322,9 +1337,9 @@ def compile(self, request: Request):
13221337

13231338
request_data = request.method_data[self.modality]
13241339
# check if sharding required
1325-
if self.config.num_sharding > 1:
1340+
if self.meta["get_n_layers"] >= self.config.num_sharding > 1:
13261341
SplitGraph, setting = model_sharding.get_split_graph_pass(
1327-
self.num_layers,
1342+
self.meta["get_n_layers"],
13281343
shares=self.config.num_sharding,
13291344
pattern=self._get_sharding_get_pattern(),
13301345
)
@@ -1340,6 +1355,14 @@ def compile(self, request: Request):
13401355
self.passes_job[TagQuantIO][QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY][
13411356
"get_quant_io_dtype_fn"
13421357
] = partial(self._tag_ios, fixed_point_type=fixed_point_type)
1358+
else:
1359+
logging.info(
1360+
f"Disabling sharding for {self.modality} because the requested number "
1361+
f"of shards ({self.config.num_sharding}) is not valid for the number "
1362+
f"of layers ({self.meta['get_n_layers']}). "
1363+
"Sharding is enabled only when the number of shards is greater than 1 "
1364+
"and no greater than the number of layers."
1365+
)
13431366

13441367
edge_prog_mgr = to_edge_transform_and_lower_to_qnn(
13451368
module=self.model,

0 commit comments

Comments
 (0)