Skip to content

Commit d614f9d

Browse files
authored
Add the MLX backend to the Apple frameworks and SwiftPM package (pytorch#22203)
1 parent 2e873cd commit d614f9d

13 files changed

Lines changed: 446 additions & 18 deletions

File tree

‎.Package.swift/backend_mlx/dummy.swift‎

Whitespace-only changes.

‎.Package.swift/backend_mlx_debug/dummy.swift‎

Whitespace-only changes.

‎.Package.swift/backend_mlx_resources/dummy.swift‎

Whitespace-only changes.

‎.github/workflows/apple.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ jobs:
192192
"executorch"
193193
"executorch_llm"
194194
"backend_coreml"
195+
"backend_mlx"
195196
"backend_xnnpack"
196197
"kernels_llm"
197198
"kernels_optimized"
@@ -220,6 +221,13 @@ jobs:
220221
zip -r "${RUNNER_TEMP}/artifacts/${FRAMEWORK}_debug-${VERSION}.zip" "${FRAMEWORK}_debug.xcframework"
221222
) done
222223
224+
# MLX metallibs are data files, so carry them in the artifact for the SwiftPM job to commit.
225+
if [ -d .Package.swift/backend_mlx_resources ]; then
226+
mkdir -p "${RUNNER_TEMP}/artifacts/backend_mlx_resources"
227+
cp .Package.swift/backend_mlx_resources/*.metallib \
228+
"${RUNNER_TEMP}/artifacts/backend_mlx_resources/" 2>/dev/null || true
229+
fi
230+
223231
upload-frameworks-ios:
224232
# NB: Don't run this on fork PRs because they won't have access to the secret and would fail anyway
225233
if: ${{ !github.event.pull_request.head.repo.fork }}
@@ -328,9 +336,38 @@ jobs:
328336
exit 1
329337
fi
330338
done < "${RUNNER_TEMP}/checksums.txt"
339+
# If MLX is being published, its Metal kernels have to come with it.
340+
# The copy below is conditional and the capture upstream is best
341+
# effort, so without this an incomplete set publishes a manifest
342+
# declaring three resources that are not on the branch, and the
343+
# consumer only finds out at Metal device init.
344+
if grep -q "^backend_mlx " "${RUNNER_TEMP}/checksums.txt"; then
345+
for SLICE in mlx-ios mlx-ios-simulator mlx-macos; do
346+
METALLIB="${RUNNER_TEMP}/frameworks-ios/backend_mlx_resources/${SLICE}.metallib"
347+
if [ ! -s "${METALLIB}" ]; then
348+
echo "::error::backend_mlx is being published but ${SLICE}.metallib is missing or empty"
349+
exit 1
350+
fi
351+
done
352+
fi
353+
# Nothing else evaluates this manifest before consumers do: the checks
354+
# above are text greps, the swiftpm branch has no CI of its own, and a
355+
# .template is not a manifest. Resolve it here so a Swift-level defect
356+
# fails this run rather than every consumer of the published branch.
357+
if ! swift package dump-package > /dev/null; then
358+
echo "::error::the substituted Package.swift does not evaluate"
359+
exit 1
360+
fi
331361
332362
git config --global user.name "PyTorch Bot"
333363
git config --global user.email "pytorchbot@users.noreply.github.com"
364+
# Commit the MLX metallibs onto the package branch beside the manifest.
365+
if [ -d "${RUNNER_TEMP}/frameworks-ios/backend_mlx_resources" ]; then
366+
mkdir -p .Package.swift/backend_mlx_resources
367+
cp "${RUNNER_TEMP}"/frameworks-ios/backend_mlx_resources/*.metallib \
368+
.Package.swift/backend_mlx_resources/
369+
git add -f .Package.swift/backend_mlx_resources/*.metallib
370+
fi
334371
git add Package.swift
335372
git commit -am "${VERSION}"
336373
git push -f origin "${BRANCH}"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,4 @@ zephyr_dev_root.backup.*/
101101
# Agents
102102
.claude/*.local.*
103103
extension/pybindings/mlx.metallib
104+
.Package.swift/backend_mlx_resources/*.metallib

‎CMakePresets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/third-party/ios-cmake/ios.toolchain.cmake",
4646
"EXECUTORCH_BUILD_PRESET_FILE": "${sourceDir}/tools/cmake/preset/macos.cmake",
4747
"PLATFORM": "MAC_ARM64",
48-
"DEPLOYMENT_TARGET": "12.0",
48+
"DEPLOYMENT_TARGET": "14.0",
4949
"CMAKE_MACOSX_BUNDLE": "OFF"
5050
},
5151
"condition": {

‎Package.swift‎

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@ let products = deliverables([
5353
"sqlite3",
5454
],
5555
],
56+
"backend_mlx": [
57+
"frameworks": [
58+
"Metal",
59+
"Foundation",
60+
"QuartzCore",
61+
],
62+
],
5663
"backend_xnnpack": [
5764
"targets": [
5865
"threadpool",
@@ -107,17 +114,49 @@ for (key, value) in products {
107114
name: key,
108115
path: "cmake-out/\(key).xcframework"
109116
))
110-
let target: Target = .target(
117+
// Split into typed locals to stay under the manifest type-check budget.
118+
let targetNames: [String] = [key] + (value["targets"] as? [String] ?? []).map {
119+
key.hasSuffix(debug_suffix) ? $0 + debug_suffix : $0
120+
}
121+
let dependencies: [Target.Dependency] = targetNames.map { .target(name: $0) }
122+
let frameworks: [LinkerSetting] =
123+
(value["frameworks"] as? [String] ?? []).map { .linkedFramework($0) }
124+
let libraries: [LinkerSetting] =
125+
(value["libraries"] as? [String] ?? []).map { .linkedLibrary($0) }
126+
packageTargets.append(.target(
111127
name: "\(key)\(dependencies_suffix)",
112-
dependencies: ([key] + (value["targets"] as? [String] ?? []).map {
113-
key.hasSuffix(debug_suffix) ? $0 + debug_suffix : $0
114-
}).map { .target(name: $0) },
128+
dependencies: dependencies,
115129
path: ".Package.swift/\(key)",
116-
linkerSettings:
117-
(value["frameworks"] as? [String] ?? []).map { .linkedFramework($0) } +
118-
(value["libraries"] as? [String] ?? []).map { .linkedLibrary($0) }
119-
)
120-
packageTargets.append(target)
130+
linkerSettings: frameworks + libraries
131+
))
132+
}
133+
134+
// One resource bundle shared by both MLX products (release and debug) so they
135+
// resolve the same bundle name, with a per-slice metallib since one slice's does
136+
// not load on another. Kept out of the loop above to avoid a per-debug copy.
137+
let mlxMetallibSlices = ["mlx-ios", "mlx-ios-simulator", "mlx-macos"]
138+
// Anchor to this file's own directory: as a dependency the manifest runs with the
139+
// consumer's cwd, where a relative path would miss every slice and ship an empty bundle.
140+
let mlxResourcesRelDir = ".Package.swift/backend_mlx_resources"
141+
let mlxResourcesDir =
142+
URL(fileURLWithPath: #filePath).deletingLastPathComponent()
143+
.appendingPathComponent(mlxResourcesRelDir).path
144+
if products.keys.contains("backend_mlx") {
145+
packageTargets.append(.target(
146+
name: "backend_mlx_resources",
147+
path: mlxResourcesRelDir,
148+
resources: mlxMetallibSlices.compactMap { slice in
149+
FileManager.default.fileExists(atPath: "\(mlxResourcesDir)/\(slice).metallib")
150+
? .copy("\(slice).metallib") : nil
151+
}
152+
))
153+
for suffix in ["", debug_suffix] {
154+
if let index = packageTargets.firstIndex(where: {
155+
$0.name == "backend_mlx\(suffix)\(dependencies_suffix)"
156+
}) {
157+
packageTargets[index].dependencies.append(.target(name: "backend_mlx_resources"))
158+
}
159+
}
121160
}
122161

123162
// Test fixtures. add_coreml.pte and add_mul_coreml.pte are generated at CI
@@ -151,6 +190,12 @@ if FileManager.default.fileExists(atPath: "\(objcTestsDir)/add_mul_coreml.pte")
151190
}
152191

153192
let testLinkerSettings: [LinkerSetting] = [
193+
// The test targets depend on the executorch binary target directly, which
194+
// carries no linker settings, rather than the with-dependencies target that
195+
// owns the libc++ link, so they must link libc++ themselves. Below a macOS 13
196+
// deployment target a Swift back-deployment shim used to supply it implicitly;
197+
// at this package's floor that shim is gone, so name it explicitly here.
198+
.linkedLibrary("c++"),
154199
.unsafeFlags([
155200
"-Xlinker", "-force_load",
156201
"-Xlinker", "cmake-out/kernels_optimized.xcframework/macos-arm64/libkernels_optimized_macos.a",
@@ -166,7 +211,7 @@ let package = Package(
166211
name: "executorch",
167212
platforms: [
168213
.iOS(.v17),
169-
.macOS(.v12),
214+
.macOS(.v14),
170215
],
171216
products: packageProducts,
172217
targets: packageTargets + [

‎backends/mlx/CMakeLists.txt‎

Lines changed: 124 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -188,16 +188,91 @@ set(_mlx_patches
188188
${CMAKE_CURRENT_SOURCE_DIR}/patches/mlx_nax_jit_sdk_gate.patch
189189
${CMAKE_CURRENT_SOURCE_DIR}/patches/mlx_qmm_splitk_bk_align.patch
190190
${CMAKE_CURRENT_SOURCE_DIR}/patches/mlx_gather_mm_rhs_lda.patch
191+
${CMAKE_CURRENT_SOURCE_DIR}/patches/mlx_metal_sdk_per_platform.patch
192+
${CMAKE_CURRENT_SOURCE_DIR}/patches/mlx_swiftpm_metallib_name.patch
191193
)
194+
# In a framework build the delegate is static, so MLX cannot find a colocated
195+
# metallib and instead loads one from a SwiftPM resource bundle. SWIFTPM_BUNDLE
196+
# is the bundle name and MLX_SWIFTPM_METALLIB_NAME the per-slice file inside it
197+
# (one per slice, since a slice's metallib does not load on another). PLATFORM
198+
# is set only for the Apple presets; a plain wheel build leaves it empty and
199+
# keeps the colocated path.
200+
set(_mlx_extra_cxx_flags "")
201+
set(_mlx_cxx_flags_arg "")
202+
if(PLATFORM)
203+
if(PLATFORM STREQUAL "OS64")
204+
set(_mlx_metallib_slice "ios")
205+
elseif(PLATFORM STREQUAL "SIMULATORARM64")
206+
set(_mlx_metallib_slice "ios-simulator")
207+
else()
208+
set(_mlx_metallib_slice "macos")
209+
endif()
210+
set(_mlx_extra_cxx_flags
211+
"-DSWIFTPM_BUNDLE=\\\"executorch_backend_mlx_resources\\\" -DMLX_SWIFTPM_METALLIB_NAME=\\\"mlx-${_mlx_metallib_slice}\\\""
212+
)
213+
# Only override the sub-build's CXX flags when there is something to add. An
214+
# empty -DCMAKE_CXX_FLAGS= on the command line beats the environment, so
215+
# passing it unconditionally would silently drop a wheel build's CXXFLAGS for
216+
# MLX only.
217+
set(_mlx_cxx_flags_arg "-DCMAKE_CXX_FLAGS=${_mlx_extra_cxx_flags}")
218+
endif()
219+
220+
# Prefer the preset's per-slice DEPLOYMENT_TARGET; fall back to the toolchain
221+
# value.
222+
if(DEPLOYMENT_TARGET)
223+
set(_mlx_osx_deployment_target ${DEPLOYMENT_TARGET})
224+
else()
225+
set(_mlx_osx_deployment_target ${CMAKE_OSX_DEPLOYMENT_TARGET})
226+
endif()
227+
228+
# The sub-build is forced to a single-config generator (Unix Makefiles), which
229+
# reads CMAKE_BUILD_TYPE, not the --config passed to the parent. Follow the
230+
# active configuration through $<CONFIG>, which resolves to the config being
231+
# built under a multi-config parent (Xcode) and to CMAKE_BUILD_TYPE under a
232+
# single-config one. It is empty only for a bare single-config parent that set
233+
# no type, so default that case to Release rather than compile MLX with no
234+
# optimization.
235+
set(_mlx_build_type "$<IF:$<BOOL:$<CONFIG>>,$<CONFIG>,Release>")
236+
237+
# Decide the MLX sub-build's build command, consumed by BUILD_COMMAND below. An
238+
# empty value leaves ExternalProject's default in place, which is only the right
239+
# choice for a Makefiles parent: ExternalProject emits `$(MAKE)` (inheriting the
240+
# parent's jobserver) only when the PARENT generator matches "Make". Under Xcode
241+
# or Ninja it emits a bare `cmake --build`, with no -j and no jobserver, so MLX
242+
# would compile on one core. Name a job count there; a bare --parallel with no
243+
# number would expand to `make -j`, unbounded.
244+
cmake_host_system_information(
245+
RESULT _mlx_host_cores QUERY NUMBER_OF_LOGICAL_CORES
246+
)
247+
if(CMAKE_GENERATOR MATCHES "Make")
248+
set(_mlx_build_command "")
249+
else()
250+
set(_mlx_build_command ${CMAKE_COMMAND} --build <BINARY_DIR> --parallel
251+
${_mlx_host_cores}
252+
)
253+
endif()
254+
192255
ExternalProject_Add(
193256
mlx_external
194257
SOURCE_DIR ${MLX_SOURCE_DIR}
195258
BINARY_DIR ${_mlx_binary_dir}
196-
PATCH_COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/patches/apply.sh
197-
${MLX_SOURCE_DIR} ${_mlx_patches}
198-
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
259+
# Force a single-config generator for the sub-build. The parent Apple build
260+
# uses the Xcode generator, which is multi-config and would place libmlx.a in
261+
# a per-config subdirectory (mlx/Debug/libmlx.a). The paths below that consume
262+
# the archive expect it flat at ${_mlx_binary_dir}/libmlx.a, the way a
263+
# single-config generator emits it, which is also how the wheel build already
264+
# produces it.
265+
CMAKE_GENERATOR "Unix Makefiles"
266+
CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${_mlx_build_type}"
199267
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
200-
-DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}
268+
${_mlx_cxx_flags_arg}
269+
# The preset's DEPLOYMENT_TARGET carries the correct per-slice
270+
# minimum (iOS 17, macOS 14); the ios.toolchain leaves
271+
# CMAKE_OSX_DEPLOYMENT_TARGET at 12.0 on the iOS slices and only
272+
# syncs it for MAC. The shader-flag patch reads
273+
# CMAKE_OSX_DEPLOYMENT_TARGET, so feed the preset value in as that,
274+
# or the iOS metallib is stamped -mios-version-min=12.0.
275+
-DCMAKE_OSX_DEPLOYMENT_TARGET=${_mlx_osx_deployment_target}
201276
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
202277
-DPLATFORM=${PLATFORM}
203278
-DDEPLOYMENT_TARGET=${DEPLOYMENT_TARGET}
@@ -217,6 +292,14 @@ ExternalProject_Add(
217292
# MLX's own install() does not emit libmlx.a where we consume it or the
218293
# metallib at all, so skip the install step and read both from the build tree.
219294
INSTALL_COMMAND ""
295+
# Build in parallel, but only where it is needed and always with a job count.
296+
# Under the multi-config Xcode parent the generated sub-build step is `cmake
297+
# --build . --config <cfg>` with no -j and no jobserver to inherit, so MLX
298+
# would compile fully serially. Under a single-config parent the default build
299+
# command is `$(MAKE)`, which already inherits the parent's jobserver, so
300+
# overriding it there would break that and serialize MLX instead. A bare
301+
# --parallel with no number expands to `make -j`, unbounded, so name a count.
302+
BUILD_COMMAND ${_mlx_build_command}
220303
# ExternalProject stamps its build, so a bare MLX submodule bump (git
221304
# submodule update) would not invalidate the stamp and we'd link a stale
222305
# libmlx.a with no signal. BUILD_ALWAYS reruns the build step every configure;
@@ -226,6 +309,25 @@ ExternalProject_Add(
226309
BUILD_BYPRODUCTS ${_mlx_static_lib} ${_mlx_metallib}
227310
)
228311

312+
# ExternalProject stamps the patch step and BUILD_ALWAYS does not re-run it, so
313+
# a reused build directory whose MLX source was reset (patches reverted) would
314+
# recompile an unpatched MLX and silently drop the iOS Metal SDK selection and
315+
# the SwiftPM metallib name. Re-apply the patches on every build; apply.sh is
316+
# idempotent (it reverse-checks each patch, skipping those already applied).
317+
# DEPENDERS configure, not build: mlx_metal_sdk_per_platform.patch edits the
318+
# sub-project's own CMake to pick the Metal SDK from PLATFORM, so it must land
319+
# before the sub-configure runs, or the shaders are built against the macOS SDK
320+
# for every slice. That does mean the sub-configure re-runs on each build, which
321+
# is the price of the ordering.
322+
ExternalProject_Add_Step(
323+
mlx_external reapply_patches
324+
COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/patches/apply.sh ${MLX_SOURCE_DIR}
325+
${_mlx_patches}
326+
DEPENDEES patch
327+
DEPENDERS configure
328+
ALWAYS 1
329+
)
330+
229331
# Imported target for the MLX static library produced by mlx_external. A static
230332
# libmlx.a carries no transitive link deps, so re-add the frameworks MLX itself
231333
# links (mirrors third-party/mlx/CMakeLists.txt:209). CPU is OFF, so Accelerate
@@ -283,6 +385,21 @@ add_dependencies(mlxdelegate mlx_schema)
283385
# Depend on mlx_external directly so libmlx.a exists before mlxdelegate links.
284386
add_dependencies(mlxdelegate mlx_external)
285387

388+
# The Apple framework build reads each static library straight from the target
389+
# output directory (it merges them without an install step), but libmlx.a is
390+
# produced by the MLX sub-build in its own binary dir, not next to the delegate.
391+
# Copy it beside libmlxdelegate.a after the delegate builds so the framework
392+
# packaging finds it where it expects. Under the multi-config Xcode generator
393+
# the Apple presets use, TARGET_FILE_DIR resolves to the per-config directory
394+
# the packaging reads (<preset>/Release), which is what this relies on.
395+
add_custom_command(
396+
TARGET mlxdelegate
397+
POST_BUILD
398+
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${_mlx_static_lib}
399+
$<TARGET_FILE_DIR:mlxdelegate>/libmlx.a
400+
VERBATIM
401+
)
402+
286403
# Add logging flag if enabled
287404
if(ET_MLX_ENABLE_OP_LOGGING)
288405
target_compile_definitions(mlxdelegate PRIVATE ET_MLX_ENABLE_OP_LOGGING=1)
@@ -333,6 +450,9 @@ endif()
333450

334451
executorch_target_link_options_shared_lib(mlxdelegate)
335452
target_compile_options(mlxdelegate PRIVATE ${_common_compile_options})
453+
# Core tensor headers carry pre-existing narrowing conversions that trip Xcode's
454+
# -Wshorten-64-to-32 -Werror; suppress it here as XNNPACK and abseil already do.
455+
target_compile_options(mlxdelegate PRIVATE -Wno-shorten-64-to-32)
336456
if(EXECUTORCH_MLX_ENABLE_SANITIZERS)
337457
target_link_options(mlxdelegate PRIVATE ${_mlx_sanitizer_link_options})
338458
endif()

0 commit comments

Comments
 (0)