@@ -223,23 +223,32 @@ def __init__(
223223 self .tok_embedding , self .decoder = self ._prepare_model ()
224224
225225 # check if sharding required
226- if self .decoder and self .config .num_sharding > 1 :
227- layer_prefix_offsets = None
228- if self .control_args .decoder_model == "gemma4-e2b" :
229- n_self = self .gemma4_config .num_self_decoder_layers
230- layer_prefix_offsets = {
231- "model.self_decoder.layers" : 0 ,
232- "model.cross_decoder.layers" : n_self ,
233- }
234- SplitGraph , setting = model_sharding .get_split_graph_pass (
235- self .meta ["get_n_layers" ],
236- shares = self .config .num_sharding ,
237- pattern = self ._get_sharding_get_pattern (),
238- layer_prefix_offsets = layer_prefix_offsets ,
239- )
240- self .passes_job [SplitGraph ] = setting
241- self .dep_table [SplitGraph ] = [FoldQDQ ]
242- self .dep_table [TagQuantIO ] = [SplitGraph ]
226+ if self .decoder is not None :
227+ if self .meta ["get_n_layers" ] >= self .config .num_sharding > 1 :
228+ layer_prefix_offsets = None
229+ if self .control_args .decoder_model == "gemma4-e2b" :
230+ n_self = self .gemma4_config .num_self_decoder_layers
231+ layer_prefix_offsets = {
232+ "model.self_decoder.layers" : 0 ,
233+ "model.cross_decoder.layers" : n_self ,
234+ }
235+ SplitGraph , setting = model_sharding .get_split_graph_pass (
236+ self .meta ["get_n_layers" ],
237+ shares = self .config .num_sharding ,
238+ pattern = self ._get_sharding_get_pattern (),
239+ layer_prefix_offsets = layer_prefix_offsets ,
240+ )
241+ self .passes_job [SplitGraph ] = setting
242+ self .dep_table [SplitGraph ] = [FoldQDQ ]
243+ self .dep_table [TagQuantIO ] = [SplitGraph ]
244+ else :
245+ logging .info (
246+ f"Disabling sharding because the requested number of shards "
247+ f"({ self .config .num_sharding } ) is not valid for the number of layers "
248+ f"({ self .meta ['get_n_layers' ]} ). "
249+ "Sharding is enabled only when the number of shards is greater than 1 "
250+ "and no greater than the number of layers."
251+ )
243252
244253 self ._decoder_inference = (
245254 DecoderInference (
@@ -1253,20 +1262,23 @@ def __init__(
12531262 if config := getattr (config , modality , None ):
12541263 if modality == AUDIO_ENCODER :
12551264 auto_model = AutoModelForSpeechSeq2Seq .from_pretrained (repo_id )
1256- self .num_layers = auto_model .config .encoder_config .num_layers
1257- self .ctx_size = auto_model .config .encoder_config .context_size
12581265 elif modality == TEXT_ENCODER :
12591266 raise NotImplementedError (f"{ modality } is under development" )
12601267 elif modality == VISION_ENCODER :
12611268 auto_model = AutoModel .from_pretrained (
12621269 repo_id , _attn_implementation = "eager"
12631270 )
1264- self .num_layers = auto_model .config .vision_config .num_hidden_layers
12651271 else :
12661272 raise NotImplementedError (f"Find no { modality } " )
12671273
12681274 auto_model = auto_model .to (torch .float32 ).eval ()
12691275 self .model = config ().create_encoder (auto_model .config ).eval ()
1276+ self .meta = self .model .get_metadata ()
1277+ if "get_n_layers" not in self .meta :
1278+ raise ValueError (
1279+ f"{ type (self .model ).__name__ } .get_metadata() must provide "
1280+ "'get_n_layers'"
1281+ )
12701282 self .model .load_state_dict (
12711283 auto_model .state_dict (), strict = False
12721284 ) # set strict to false to simplify parameter loading for non-text models
@@ -1298,12 +1310,15 @@ def _tag_ios(self, node, fixed_point_type):
12981310 quant_io_type = fixed_point_type ["io_type" ]
12991311
13001312 # GraniteSpeech: tag _to_copy op as quantized tensors for attn dist. It is caused by sharding
1313+ ctx_size = self .meta .get ("get_context_size" )
13011314 if (
13021315 issubclass (self .config , GraniteSpeechEncoder )
13031316 and node .target == exir_ops .edge .aten ._to_copy .default
1304- and node .meta ["val" ].size () == (self .ctx_size , self .ctx_size )
1317+ and ctx_size is not None
1318+ and node .meta ["val" ].size () == (ctx_size , ctx_size )
13051319 ):
13061320 quant_io_type = torch .int32
1321+ logging .info (f"Tagging { node } with { quant_io_type } quantized IO " )
13071322
13081323 return quant_io_type
13091324
@@ -1322,9 +1337,9 @@ def compile(self, request: Request):
13221337
13231338 request_data = request .method_data [self .modality ]
13241339 # check if sharding required
1325- if self .config .num_sharding > 1 :
1340+ if self .meta [ "get_n_layers" ] >= self . config .num_sharding > 1 :
13261341 SplitGraph , setting = model_sharding .get_split_graph_pass (
1327- self .num_layers ,
1342+ self .meta [ "get_n_layers" ] ,
13281343 shares = self .config .num_sharding ,
13291344 pattern = self ._get_sharding_get_pattern (),
13301345 )
@@ -1340,6 +1355,14 @@ def compile(self, request: Request):
13401355 self .passes_job [TagQuantIO ][QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY ][
13411356 "get_quant_io_dtype_fn"
13421357 ] = partial (self ._tag_ios , fixed_point_type = fixed_point_type )
1358+ else :
1359+ logging .info (
1360+ f"Disabling sharding for { self .modality } because the requested number "
1361+ f"of shards ({ self .config .num_sharding } ) is not valid for the number "
1362+ f"of layers ({ self .meta ['get_n_layers' ]} ). "
1363+ "Sharding is enabled only when the number of shards is greater than 1 "
1364+ "and no greater than the number of layers."
1365+ )
13431366
13441367 edge_prog_mgr = to_edge_transform_and_lower_to_qnn (
13451368 module = self .model ,
0 commit comments