Skip to content

Commit 8ab593b

Browse files
authored
Add prefill/decode multifunction support in ET (pytorch#16552)
Differential Revision: D91243088 Pull Request resolved: pytorch#16791
1 parent 8e70dd2 commit 8ab593b

17 files changed

Lines changed: 2270 additions & 232 deletions

‎backends/apple/coreml/compiler/coreml_preprocess.py‎

Lines changed: 430 additions & 63 deletions
Large diffs are not rendered by default.

‎backends/apple/coreml/runtime/delegate/ETCoreMLModel.mm‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,12 +200,27 @@ - (nullable instancetype)initWithAsset:(ETCoreMLAsset *)asset
200200
return nil;
201201
}
202202

203+
if (orderedInputNames == nil) {
204+
ETCoreMLLogErrorAndSetNSError(error,
205+
ETCoreMLErrorCorruptedModel,
206+
"orderedInputNames must not be nil");
207+
return nil;
208+
}
209+
210+
if (orderedOutputNames == nil) {
211+
ETCoreMLLogErrorAndSetNSError(error,
212+
ETCoreMLErrorCorruptedModel,
213+
"orderedOutputNames must not be nil");
214+
return nil;
215+
}
216+
203217
self = [super init];
204218
if (self) {
205219
_mlModel = mlModel;
206220
_asset = asset;
207221
_orderedInputNames = [orderedInputNames copy];
208222
_orderedOutputNames = [orderedOutputNames copy];
223+
209224
_cache = [[NSCache alloc] init];
210225
_inputConstraintsByName = get_multi_array_input_constraints_by_name(mlModel.modelDescription);
211226
_outputConstraintsByName = get_multi_array_output_constraints_by_name(mlModel.modelDescription);
@@ -234,6 +249,15 @@ - (NSString *)identifier {
234249
BOOL lCopyData = copyData;
235250
NSString *argName = [nameEnumerator nextObject];
236251
MLMultiArrayConstraint *constraint = argConstraintsByName[argName];
252+
253+
if (constraint == nil) {
254+
ETCoreMLLogErrorAndSetNSError(error,
255+
ETCoreMLErrorCorruptedModel,
256+
"No constraint found for arg '%@'. Model may have mismatched input/output names.",
257+
argName);
258+
return nil;
259+
}
260+
237261
const auto& layout = arg.layout();
238262
auto dataType = to_ml_multiarray_data_type(layout.dataType());
239263
MLMultiArray *multiArrayArg = nil;

‎backends/apple/coreml/runtime/delegate/ETCoreMLModelLoader.mm‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,6 @@
2626

2727
return result;
2828
}
29-
30-
ETCoreMLModel * _Nullable get_model_from_asset(ETCoreMLAsset *asset,
31-
MLModelConfiguration *configuration,
32-
const executorchcoreml::ModelMetadata& metadata,
33-
NSError * __autoreleasing *error) {
34-
NSOrderedSet<NSString *> *orderedInputNames = ::get_ordered_set(metadata.input_names);
35-
NSOrderedSet<NSString *> *orderedOutputNames = ::get_ordered_set(metadata.output_names);
36-
ETCoreMLModel *model = [[ETCoreMLModel alloc] initWithAsset:asset
37-
configuration:configuration
38-
orderedInputNames:orderedInputNames
39-
orderedOutputNames:orderedOutputNames
40-
error:error];
41-
return model;
42-
}
4329
} // namespace
4430

4531
@implementation ETCoreMLModelLoader
@@ -48,14 +34,30 @@ + (nullable ETCoreMLModel *)loadModelWithCompiledAsset:(ETCoreMLAsset *)compiled
4834
configuration:(MLModelConfiguration *)configuration
4935
metadata:(const executorchcoreml::ModelMetadata&)metadata
5036
error:(NSError * __autoreleasing *)error {
37+
if (compiledAsset == nil) {
38+
return nil;
39+
}
40+
41+
// Use the metadata's ordered input/output names.
42+
// For multifunction models, the caller should load the per-method metadata
43+
// which contains the correct input/output names for that method.
44+
NSOrderedSet<NSString *> *orderedInputNames = ::get_ordered_set(metadata.input_names);
45+
NSOrderedSet<NSString *> *orderedOutputNames = ::get_ordered_set(metadata.output_names);
46+
5147
NSError *localError = nil;
52-
ETCoreMLModel *model = (compiledAsset != nil) ? get_model_from_asset(compiledAsset, configuration, metadata, &localError) : nil;
48+
ETCoreMLModel *model = [[ETCoreMLModel alloc] initWithAsset:compiledAsset
49+
configuration:configuration
50+
orderedInputNames:orderedInputNames
51+
orderedOutputNames:orderedOutputNames
52+
error:&localError];
5353
if (model) {
5454
return model;
5555
}
56+
5657
if (error) {
5758
*error = localError;
5859
}
60+
5961
return nil;
6062
}
6163

‎backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,27 @@ __attribute__((objc_subclassing_restricted))
5050
configuration:(MLModelConfiguration*)configuration
5151
error:(NSError* __autoreleasing*)error;
5252

53+
/// Loads the model from the AOT data with an optional method name for multifunction model support.
54+
///
55+
/// The data is the AOT blob stored in the executorch Program. The method first parses the model
56+
/// metadata stored in the blob and extracts the identifier. For multifunction CoreML models,
57+
/// the methodName is used to populate the correct input/output names from the method's metadata,
58+
/// while functionName specifies the CoreML function to invoke (via MLModelConfiguration.functionName).
59+
/// The method/function names are NOT included in the cache key since the multifunction
60+
/// model contains all functions and should only be compiled once.
61+
///
62+
/// @param data The AOT blob data.
63+
/// @param configuration The model configuration that will be used to load the model.
64+
/// @param methodName Optional method name (e.g., "forward", "prefill") for metadata lookup.
65+
/// @param functionName Optional CoreML function name to invoke. If nil, methodName is used.
66+
/// @param error On failure, error is filled with the failure information.
67+
/// @retval An opaque handle that points to the loaded model.
68+
- (ModelHandle*)loadModelFromAOTData:(NSData*)data
69+
configuration:(MLModelConfiguration*)configuration
70+
methodName:(nullable NSString*)methodName
71+
functionName:(nullable NSString*)functionName
72+
error:(NSError* __autoreleasing*)error;
73+
5374
/// Executes the loaded model.
5475
///
5576
/// @param handle The handle to the loaded model.

0 commit comments

Comments
 (0)