Skip to content

Commit b928496

Browse files
authored
Remove aoti_cuda/metal_blob.ptd file from being generated for AOTI (pytorch#16809)
As titled
1 parent c7c4073 commit b928496

7 files changed

Lines changed: 71 additions & 26 deletions

File tree

‎.ci/scripts/export_model_artifact.sh‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,10 @@ if [ "$MODEL_NAME" = "parakeet" ]; then
164164
--output-dir "${OUTPUT_DIR}"
165165

166166
test -f "${OUTPUT_DIR}/model.pte"
167-
test -f "${OUTPUT_DIR}/aoti_${DEVICE}_blob.ptd"
167+
# CUDA saves named data to separate .ptd file, Metal embeds in .pte
168+
if [ "$DEVICE" = "cuda" ]; then
169+
test -f "${OUTPUT_DIR}/aoti_cuda_blob.ptd"
170+
fi
168171
test -f "${OUTPUT_DIR}/tokenizer.model"
169172
ls -al "${OUTPUT_DIR}"
170173
echo "::endgroup::"
@@ -200,7 +203,10 @@ if [ -n "$PREPROCESSOR_OUTPUT" ]; then
200203
fi
201204

202205
test -f model.pte
203-
test -f aoti_${DEVICE}_blob.ptd
206+
# CUDA saves named data to separate .ptd file, Metal embeds in .pte
207+
if [ "$DEVICE" = "cuda" ]; then
208+
test -f aoti_cuda_blob.ptd
209+
fi
204210
if [ -n "$PREPROCESSOR_OUTPUT" ]; then
205211
test -f $PREPROCESSOR_OUTPUT
206212
fi
@@ -209,7 +215,10 @@ echo "::endgroup::"
209215
echo "::group::Store $MODEL_NAME Artifacts"
210216
mkdir -p "${OUTPUT_DIR}"
211217
mv model.pte "${OUTPUT_DIR}/"
212-
mv aoti_${DEVICE}_blob.ptd "${OUTPUT_DIR}/"
218+
# CUDA saves named data to separate .ptd file, Metal embeds in .pte
219+
if [ "$DEVICE" = "cuda" ]; then
220+
mv aoti_cuda_blob.ptd "${OUTPUT_DIR}/"
221+
fi
213222
if [ -n "$PREPROCESSOR_OUTPUT" ]; then
214223
mv $PREPROCESSOR_OUTPUT "${OUTPUT_DIR}/"
215224
fi

‎.ci/scripts/test_model_e2e.sh‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ Arguments:
3030
- quantized-int4-weight-only
3131
3232
model_dir Directory containing model artifacts (optional, default: current directory)
33-
Expected files: model.pte, aoti_cuda_blob.ptd/aoti_metal_blob.ptd
33+
Expected files: model.pte, aoti_cuda_blob.ptd (CUDA only)
3434
Tokenizers and test files will be downloaded to this directory
3535
3636
Examples:
@@ -67,13 +67,14 @@ MODEL_DIR="${4:-.}"
6767

6868
echo "Testing model: $HF_MODEL (quantization: $QUANT_NAME)"
6969

70-
# Make sure model.pte and aoti_${DEVICE}_blob.ptd exist
70+
# Make sure model.pte exists
7171
if [ ! -f "$MODEL_DIR/model.pte" ]; then
7272
echo "Error: model.pte not found in $MODEL_DIR"
7373
exit 1
7474
fi
75-
if [ ! -f "$MODEL_DIR/aoti_${DEVICE}_blob.ptd" ]; then
76-
echo "Error: aoti_${DEVICE}_blob.ptd not found in $MODEL_DIR"
75+
# For CUDA, also check for aoti_cuda_blob.ptd (Metal embeds data in .pte)
76+
if [ "$DEVICE" = "cuda" ] && [ ! -f "$MODEL_DIR/aoti_cuda_blob.ptd" ]; then
77+
echo "Error: aoti_cuda_blob.ptd not found in $MODEL_DIR"
7778
exit 1
7879
fi
7980
# Locate EXECUTORCH_ROOT from the directory of this script
@@ -190,7 +191,11 @@ fi
190191

191192
# Build runner command with common arguments
192193
RUNNER_BIN="cmake-out/examples/models/$RUNNER_PATH/$RUNNER_TARGET"
193-
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_${DEVICE}_blob.ptd --temperature 0"
194+
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --temperature 0"
195+
# For CUDA, add data_path argument (Metal embeds data in .pte)
196+
if [ "$DEVICE" = "cuda" ]; then
197+
RUNNER_ARGS="$RUNNER_ARGS --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd"
198+
fi
194199

195200
# Add model-specific arguments
196201
case "$MODEL_NAME" in
@@ -204,7 +209,11 @@ case "$MODEL_NAME" in
204209
RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/ --image_path $IMAGE_PATH"
205210
;;
206211
parakeet)
207-
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_${DEVICE}_blob.ptd --audio_path ${MODEL_DIR}/$AUDIO_FILE --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE"
212+
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --audio_path ${MODEL_DIR}/$AUDIO_FILE --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE"
213+
# For CUDA, add data_path argument (Metal embeds data in .pte)
214+
if [ "$DEVICE" = "cuda" ]; then
215+
RUNNER_ARGS="$RUNNER_ARGS --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd"
216+
fi
208217
;;
209218
esac
210219

‎backends/aoti/aoti_backend.py‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,19 @@ def get_custom_passes(cls, compile_specs: List[CompileSpec]) -> List[typing.Any]
7474
"""Return the list of custom passes to apply after ReplaceViewCopyWithViewPass and before decomposition."""
7575
pass
7676

77+
@classmethod
78+
def save_data_externally(cls) -> bool:
79+
"""
80+
Return whether to save the named data map to an external .ptd file.
81+
82+
If True, the SO blob and weights blob will be saved to a separate .ptd file
83+
(e.g., aoti_cuda_blob.ptd) that must be provided at runtime.
84+
If False, the data will be merged into the .pte file.
85+
86+
Default is False (merge with .pte file). Subclasses can override this.
87+
"""
88+
return False
89+
7790
@classmethod
7891
def get_extra_aoti_compile_context_manager(cls):
7992
"""Return extra context manager to apply during aoti_compile stage. By default returns an empty context manager."""
@@ -225,11 +238,15 @@ def preprocess(
225238
named_data_store = NamedDataStore()
226239
method_name = cls.method_name_from_compile_specs(compile_specs)
227240

228-
# Add SO and weights blob separately
229241
named_data_store.add_named_data(method_name + "_so_blob", so_data, 1, None)
230-
weights_blob_data_type = f"aoti_{device_name}_blob"
242+
# Determine whether to save named data externally based on backend setting
243+
# External: save to separate .ptd file, otherwise merge with .pte file
244+
external_tag = (
245+
f"aoti_{device_name}_blob" if cls.save_data_externally() else None
246+
)
247+
231248
named_data_store.add_named_data(
232-
method_name + "_weights_blob", blob_data, 1, weights_blob_data_type
249+
method_name + "_weights_blob", blob_data, 1, external_tag
233250
)
234251

235252
# Clean up the generated files

‎backends/cuda/cuda_backend.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,14 @@ def _setup_cuda_environment_for_fatbin() -> bool:
125125
except Exception:
126126
return False
127127

128+
@classmethod
129+
def save_data_externally(cls) -> bool:
130+
"""
131+
CUDA backend saves SO blob and weights blob to an external .ptd file.
132+
This file must be provided at runtime via --data_path argument.
133+
"""
134+
return True
135+
128136
@classmethod
129137
def get_supported_fallback_kernels(cls) -> Dict[str, Any]:
130138
return {

‎examples/models/parakeet/README.md‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ python export_parakeet_tdt.py --backend metal --output-dir ./parakeet_metal
3838
```
3939

4040
This generates:
41-
- `model.pte` - The compiled Parakeet TDT model
42-
- `aoti_metal_blob.ptd` - Metal kernel blob required at runtime
41+
- `model.pte` - The compiled Parakeet TDT model (includes Metal kernel blob)
4342
- `tokenizer.model` - SentencePiece tokenizer
4443

4544
### CUDA Export (Linux)
@@ -81,10 +80,9 @@ From the executorch root directory:
8180
--audio_path /path/to/audio.wav \
8281
--tokenizer_path examples/models/parakeet/parakeet_tdt_exports/tokenizer.model
8382

84-
# Metal (include .ptd data file)
83+
# Metal
8584
DYLD_LIBRARY_PATH=/usr/lib ./cmake-out/examples/models/parakeet/parakeet_runner \
8685
--model_path examples/models/parakeet/parakeet_metal/model.pte \
87-
--data_path examples/models/parakeet/parakeet_metal/aoti_metal_blob.ptd \
8886
--audio_path /path/to/audio.wav \
8987
--tokenizer_path examples/models/parakeet/parakeet_metal/tokenizer.model
9088

@@ -103,5 +101,5 @@ DYLD_LIBRARY_PATH=/usr/lib ./cmake-out/examples/models/parakeet/parakeet_runner
103101
| `--model_path` | Path to Parakeet model (.pte) |
104102
| `--audio_path` | Path to input audio file (.wav) |
105103
| `--tokenizer_path` | Path to tokenizer file (default: `tokenizer.json`) |
106-
| `--data_path` | Path to data file (.ptd) for delegate data (optional, required for Metal/CUDA) |
104+
| `--data_path` | Path to data file (.ptd) for delegate data (required for CUDA) |
107105
| `--timestamps` | Timestamp output mode: `none\|token\|word\|segment\|all` (default: `segment`) |

‎examples/models/voxtral/README.md‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,7 @@ optimum-cli export executorch \
8989
```
9090

9191
This will generate:
92-
- `model.pte` - The exported model
93-
- `aoti_metal_blob.ptd` - The Metal kernel blob required for runtime
92+
- `model.pte` - The exported model (includes Metal kernel blob)
9493

9594
See the "Building the multimodal runner" section below for instructions on building with Metal support, and the "Running the model" section for runtime instructions.
9695

@@ -162,10 +161,17 @@ If you already have a preprocessed mel spectrogram saved as a `.bin` file, you c
162161
--audio_path path/to/preprocessed_audio.bin
163162
```
164163

165-
### Running on CUDA or Metal:
166-
Add the `--data_path` argument to provide the appropriate data blob to the commands above:
167-
- For CUDA: `--data_path path/to/aoti_cuda_blob.ptd`
168-
- For Metal: `--data_path path/to/aoti_metal_blob.ptd`
164+
### Running on CUDA:
165+
Add the `--data_path` argument to provide the CUDA data blob:
166+
```
167+
./cmake-out/examples/models/voxtral/voxtral_runner \
168+
--model_path path/to/model.pte \
169+
--data_path path/to/aoti_cuda_blob.ptd \
170+
--tokenizer_path path/to/tekken.json \
171+
--prompt "What can you tell me about this audio?" \
172+
--audio_path path/to/audio_input.wav \
173+
--processor_path path/to/voxtral_preprocessor.pte
174+
```
169175

170176
### Example output:
171177
```

‎examples/models/whisper/README.md‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,7 @@ optimum-cli export executorch \
7171
```
7272

7373
This command generates:
74-
- `model.pte` — Compiled Whisper model
75-
- `aoti_metal_blob.ptd` — Weight data file for Metal backend
74+
- `model.pte` — Compiled Whisper model (includes Metal kernel blob)
7675

7776
### Preprocessor
7877

@@ -160,7 +159,6 @@ cmake-out/examples/models/whisper/whisper_runner \
160159
# Run the Whisper runner
161160
cmake-out/examples/models/whisper/whisper_runner \
162161
--model_path model.pte \
163-
--data_path aoti_metal_blob.ptd \
164162
--tokenizer_path ./ \
165163
--audio_path output.wav \
166164
--processor_path whisper_preprocessor.pte \

0 commit comments

Comments
 (0)