[Core] Upgrade the bundled Ray to 2.56.1

Ray 2.9.3 is from January 2024 and carries CVE-2025-62593, a dashboard RCE
(CVSS 9.4, in CISA KEV) fixed in 2.52.0. 2.56.1 is the newest release that
still supports Python 3.8-3.11.

The seven patches were re-triaged against the new source rather than
regenerated blindly:

- cli.py is deleted. It rewrote `ray job submit` output, and nothing has
  called `ray job submit` since #4318.
- resource_demand_scheduler loses one of its two hunks: the `assert not
  unfulfilled` it removed is gone upstream.
- The remaining five still apply, and are regenerated against ray-2.56.1.
  Four of them (autoscaler, command_runner, resource_demand_scheduler,
  updater) only run under ProvisionerVersion.RAY_AUTOSCALER, which is IBM
  alone -- untested here, see the PR description.

monkey_patch_hash_launch_conf now delegates to Ray's own hash_launch_conf
instead of copying it. `ray up` runs against whatever Ray the client has, and
Ray has already changed this digest once (sha1/hexdigest -> sha256/base32hex);
a copy silently disagrees with the hash Ray's node_launcher writes for worker
nodes. Only the ssh_proxy_command exclusion stays ours.

Images and docs are bumped alongside, and a test asserts every such pin still
agrees with SKY_REMOTE_RAY_VERSION -- a stale one fails silently, since the
patch guards simply skip.

Author: DanielZhangQD

Dockerfile_k8s

@@ -73,7 +73,7 @@ RUN ARCH=${TARGETARCH:-$(case "$(uname -m)" in \
     $HOME/.local/bin/uv venv ~/skypilot-runtime --seed --python=3.10 && \
     source ~/skypilot-runtime/bin/activate && \
     $HOME/.local/bin/uv pip install 'skypilot-nightly[remote,kubernetes]' \
-        'ray[default]==2.9.3' 'pycryptodome==3.12.0' && \
+        'ray[default]==2.56.1' 'pycryptodome==3.12.0' && \
     $HOME/.local/bin/uv pip uninstall skypilot-nightly && \
     curl -LO "https://dl.k8s.io/release/v1.33.12/bin/linux/$ARCH/kubectl" && \
     # Install kubectl to user's local bin instead of system path to avoid

Dockerfile_k8s_gpu

@@ -78,7 +78,7 @@ RUN ARCH=${TARGETARCH:-$(case "$(uname -m)" in \
     $HOME/.local/bin/uv venv ~/skypilot-runtime --seed --python=3.10 && \
     source ~/skypilot-runtime/bin/activate && \
     $HOME/.local/bin/uv pip install 'skypilot-nightly[remote,kubernetes]' \
-        'ray[default]==2.9.3' 'pycryptodome==3.12.0' && \
+        'ray[default]==2.56.1' 'pycryptodome==3.12.0' && \
     $HOME/.local/bin/uv pip uninstall skypilot-nightly && \
     curl -LO "https://dl.k8s.io/release/v1.33.12/bin/linux/$ARCH/kubectl" && \
     # Install kubectl to user's local bin instead of system path to avoid

docs/source/reference/architecture/internals.rst

@@ -75,7 +75,7 @@ SkyPilot starts its Ray cluster when provisioning nodes:
 
 - On the head node: ``ray start --head`` with port 6380 and custom resource configurations
 - On worker nodes: ``ray start --address`` connecting to the head node's internal IP
-- Ray version is set to ``2.9.3`` in ``SKY_REMOTE_RAY_VERSION``
+- Ray version is set to ``2.56.1`` in ``SKY_REMOTE_RAY_VERSION``
 - Runtime environment: ``skypilot-runtime`` conda environment
 
 **Important considerations for users:**

docs/source/reference/kubernetes/kubernetes-getting-started.rst

@@ -454,7 +454,7 @@ FAQs
     RUN export PIP_DISABLE_PIP_VERSION_CHECK=1 && \
         python3 -m venv ~/skypilot-runtime && \
         PYTHON_EXEC=$(echo ~/skypilot-runtime)/bin/python && \
-        $PYTHON_EXEC -m pip install 'skypilot-nightly[remote,kubernetes]' 'ray[default]==2.9.3' 'pycryptodome==3.12.0' && \
+        $PYTHON_EXEC -m pip install 'skypilot-nightly[remote,kubernetes]' 'ray[default]==2.56.1' 'pycryptodome==3.12.0' && \
         $PYTHON_EXEC -m pip uninstall skypilot-nightly -y && \
         curl -LO "https://dl.k8s.io/release/v1.28.11/bin/linux/amd64/kubectl" && \
         sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \

sky/backends/monkey_patches/monkey_patch_ray_up.py

@@ -9,7 +9,7 @@
    a cluster is first created. This then makes it possible for us to support
    changing the proxy command without changing the launch hash in the future.
 
-2. The _should_create_new_head() function inside Ray autoscaler to avoid the ray up 
+2. The _should_create_new_head() function inside Ray autoscaler to avoid the ray up
 checking the launch hash when creating the head node.
 
 Reasons:
@@ -26,32 +26,21 @@
    the logic for considering what worker nodes to restart/"reuse" is in node_provider.py's
    create_node(), the same code for deciding what head node to restart/"reuse".
 """
-import hashlib
-import json
-import os
-
 from ray.autoscaler import sdk
+from ray.autoscaler._private import util as ray_autoscaler_util
 
 
-# Ref: https://github.com/ray-project/ray/blob/releases/2.4.0/python/ray/autoscaler/_private/util.py#L396-L408
+# Delegate to Ray's own implementation rather than copying it: `ray up` runs
+# against whatever Ray the client has installed, and Ray has already changed
+# this digest once (sha1/hexdigest -> sha256/base32hex). A copy would silently
+# disagree with the hash Ray's own node_launcher writes for worker nodes.
 def monkey_patch_hash_launch_conf(node_conf, auth):
-    hasher = hashlib.sha1()
-    # For hashing, we replace the path to the key with the key
-    # itself. This is to make sure the hashes are the same even if keys
-    # live at different locations on different machines.
-    full_auth = auth.copy()
-    full_auth.pop('ssh_proxy_command', None)  # NOTE: skypilot changes.
-    for key_type in ['ssh_private_key', 'ssh_public_key']:
-        if key_type in auth:
-            with open(os.path.expanduser(auth[key_type]),
-                      encoding='utf-8') as key:
-                full_auth[key_type] = key.read()
-    hasher.update(
-        json.dumps([node_conf, full_auth], sort_keys=True).encode('utf-8'))
-    return hasher.hexdigest()
+    # NOTE: skypilot change -- exclude ssh_proxy_command from the hash.
+    auth = {k: v for k, v in auth.items() if k != 'ssh_proxy_command'}
+    return ray_autoscaler_util.hash_launch_conf(node_conf, auth)
 
 
-# Ref: https://github.com/ray-project/ray/blob/releases/2.4.0/python/ray/autoscaler/_private/commands.py#L854-L912
+# Ref: https://github.com/ray-project/ray/blob/ray-2.56.1/python/ray/autoscaler/_private/commands.py#L999-L1057
 def monkey_patch_should_create_new_head(
     head_node_id,
     new_launch_hash,

sky/catalog/images/provisioners/skypilot.sh

@@ -56,14 +56,16 @@ PYTHON_EXEC=$(echo ~/skypilot-runtime)/bin/python
 $PYTHON_EXEC -m pip install "skypilot-nightly[remote]"
 
 # Install Ray
+# Keep RAY_VERSION in sync with SKY_REMOTE_RAY_VERSION in skylet.constants.
 # Pin click<8.3.0: click 8.3.0+ breaks Ray CLI due to deepcopy issues with
 # Sentinel values. See https://github.com/ray-project/ray/issues/56747.
+RAY_VERSION=2.56.1
 RAY_ADDRESS=127.0.0.1:6380
-$PYTHON_EXEC -m pip install --exists-action w -U "ray[default]==2.9.3" "click<8.3.0"
+$PYTHON_EXEC -m pip install --exists-action w -U "ray[default]==$RAY_VERSION" "click<8.3.0"
 export PATH=$PATH:$HOME/.local/bin
 source ~/skypilot-runtime/bin/activate
 which ray > ~/.sky/ray_path || exit 1
-$PYTHON_EXEC -m pip list | grep "ray " | grep 2.9.3 2>&1 > /dev/null && {
+$PYTHON_EXEC -m pip list | grep "ray " | grep $RAY_VERSION 2>&1 > /dev/null && {
   $PYTHON_EXEC -c "from sky.skylet.ray_patches import patch; patch()" || exit 1
 }
 

sky/provision/docker_utils.py

@@ -16,9 +16,10 @@
 # Configure environment variables. A docker image can have environment variables
 # set in the Dockerfile with `ENV``. We need to export these variables to the
 # shell environment, so that our ssh session can access them.
-# Filter out RAY_RUNTIME_ENV_HOOK to prevent Ray version conflicts.
-# Docker images with Ray 2.48.0+ set this for UV package manager support,
-# but it causes FAILED_DRIVER errors with SkyPilot's Ray 2.9.3.
+# Filter out RAY_RUNTIME_ENV_HOOK to prevent Ray version conflicts. Docker
+# images with Ray 2.48.0+ set this for UV package manager support, pointing at
+# the image's own Ray rather than the one in SkyPilot's runtime venv, which
+# causes FAILED_DRIVER errors.
 # See: https://github.com/skypilot-org/skypilot/pull/7181
 SETUP_ENV_VARS_CMD = (
     'prefix_cmd() '

sky/skylet/constants.py

@@ -54,7 +54,7 @@
 # i.e. the PORT_DICT_STR above.
 SKY_REMOTE_RAY_PORT_FILE = '.sky/ray_port.json'
 SKY_REMOTE_RAY_TEMPDIR = '/tmp/ray_skypilot'
-SKY_REMOTE_RAY_VERSION = '2.9.3'
+SKY_REMOTE_RAY_VERSION = '2.56.1'
 
 # To avoid user image causing issue with the SkyPilot runtime, we run SkyPilot
 # commands the following prefix:

sky/skylet/ray_patches/apply_patches.py

@@ -24,7 +24,6 @@
 _PATCHES = [
     ('ray._private.log_monitor', 'log_monitor.py.patch'),
     ('ray._private.worker', 'worker.py.patch'),
-    ('ray.dashboard.modules.job.cli', 'cli.py.patch'),
     ('ray.autoscaler._private.autoscaler', 'autoscaler.py.patch'),
     ('ray.autoscaler._private.command_runner', 'command_runner.py.patch'),
     ('ray.autoscaler._private.resource_demand_scheduler',

sky/skylet/ray_patches/autoscaler.py.diff

@@ -1,13 +1,13 @@
 --- a/autoscaler.py
 +++ b/autoscaler.py
 @@ -1,3 +1,6 @@
-+# From https://github.com/ray-project/ray/blob/ray-2.9.3/python/ray/autoscaler/_private/autoscaler.py
++# From https://github.com/ray-project/ray/blob/ray-2.56.1/python/ray/autoscaler/_private/autoscaler.py
 +# Sky patch changes:
 +#  - enable upscaling_speed to be 0.0
  import copy
  import logging
  import math
-@@ -1071,7 +1074,7 @@
+@@ -1117,7 +1120,7 @@
              upscaling_speed = self.config.get("upscaling_speed")
              aggressive = self.config.get("autoscaling_mode") == "aggressive"
              target_utilization_fraction = self.config.get("target_utilization_fraction")