Push hugging face storage resolution into vortex-cloud rather than vortex-python (#9199)

We should treat hf:// urls as any other object store, right now we are
keeping all of it in vortex-python which is not quite as versatile as
having a full fledged object store.

fixes #5379

---------

Signed-off-by: Robert Kruszewski <robert@spiraldb.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

Author: robert3005

Cargo.lock

@@ -170,6 +170,7 @@ glob = "0.3.2"
 goldenfile = "1"
 half = { version = "2.7.1", features = ["std", "num-traits"] }
 hashbrown = "0.17.1"
+http = "1.5.0"
 humansize = "2.1.3"
 indicatif = "0.18.0"
 insta = "1.43"
@@ -207,6 +208,7 @@ parquet-variant = "58.3"
 parquet-variant-compute = "58.3"
 paste = "1.0.15"
 pco = "1.0.1"
+percent-encoding = "2.3.2"
 pin-project-lite = "0.2.15"
 primitive-types = { version = "0.14.0" }
 proc-macro2 = "1.0.95"

Cargo.toml

@@ -19,8 +19,9 @@ transforms.
 
 Hub repositories are streamed in place: files are read with HTTP range requests, so only the
 projected columns and matching rows are ever transferred. Private and gated repositories
-authenticate with the ``token`` argument or the locally saved login. Files are downloaded (with
-the usual Hub caching) only when ``streaming=False`` or ``local_files_only=True``.
+authenticate with the ``token`` argument, ``HF_TOKEN``, or the locally saved login — see
+:doc:`store/huggingface` for the full precedence. Files are downloaded (with the usual Hub
+caching) only when ``streaming=False`` or ``local_files_only=True``.
 
 .. code-block:: python
 

docs/api/python/datasets.rst

@@ -11,6 +11,7 @@ Vortex arrays support reading and writing to many object storage systems:
    store/gcs
    store/azure
    store/http
+   store/huggingface
    store/local
    store/memory
    store/opendal

docs/api/python/store.rst

@@ -0,0 +1,97 @@
+==================
+Hugging Face Hub
+==================
+
+Vortex reads Hugging Face Hub repositories over ``hf://`` URLs. A Hub repository is a set of files
+behind an HTTP endpoint that honours range requests, so no cloud SDK is involved and no extra build
+feature is needed.
+
+.. list-table::
+   :header-rows: 1
+
+   * - URL
+     - Repository kind
+   * - ``hf://datasets/<owner>/<name>[@<revision>][/<path>]``
+     - Dataset
+   * - ``hf://spaces/<owner>/<name>[@<revision>][/<path>]``
+     - Space
+   * - ``hf://<owner>/<name>[@<revision>][/<path>]``
+     - Model
+
+``<revision>`` is a branch, tag or commit, defaulting to ``main``. A revision containing ``/`` must
+be percent-encoded, e.g. ``hf://datasets/org/name@refs%2Fconvert%2Fparquet/data/train.vortex``.
+
+Configuration comes from the same environment variables ``huggingface_hub`` reads:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Variable
+     - Meaning
+   * - ``HF_TOKEN``
+     - API token for private and gated repositories. Falls back to the token file at
+       ``HF_TOKEN_PATH``, then ``$HF_HOME/token``, then ``$HOME/.cache/huggingface/token``.
+   * - ``HF_ENDPOINT``
+     - Hub endpoint, defaulting to ``https://huggingface.co``.
+
+Reading from the Hub
+====================
+
+Pass an ``hf://`` URL directly. Public repositories need no credentials; private and gated ones
+authenticate from ``HF_TOKEN`` or the saved login:
+
+.. code-block:: python
+
+   import vortex as vx
+
+   vxf = vx.open("hf://datasets/org/name/data/train.vortex")
+   for batch in vxf.to_arrow():
+       ...
+
+:class:`vortex.store.HfStore`
+=============================
+
+.. py:class:: vortex.store.HfStore(repo_id, *, repo_type="dataset", revision=None, token=None, endpoint=None)
+
+   A Hugging Face Hub object store, rooted at one repository and revision.
+
+   A URL is enough for most reads, so reach for this class only for the two things a URL cannot
+   express: a token held in a variable rather than the environment, and a read that must stay
+   anonymous even though the environment offers credentials.
+
+   Because the store is rooted at the repository and revision, the path passed alongside it is a
+   path *within* the repository.
+
+   :param repo_id: The repository, as ``"<owner>/<name>"``.
+   :param repo_type: ``"dataset"``, ``"model"`` or ``"space"``. Defaults to ``"dataset"``.
+   :param revision: A branch, tag or commit. Defaults to ``main``. Unlike in a URL, a revision
+       containing ``/`` is passed literally — the store percent-encodes it.
+   :param token: ``None`` (the default) or ``True`` authenticates from ``HF_TOKEN`` or the saved
+       login; ``False`` forces an anonymous read even when credentials are available; a string is
+       used as the token directly.
+   :param endpoint: Hub endpoint. Defaults to ``HF_ENDPOINT``, then ``https://huggingface.co``.
+
+.. code-block:: python
+
+   import vortex as vx
+   from vortex.store import HfStore
+
+   store = HfStore("org/name", revision="refs/convert/parquet", token="hf_...")
+
+   # With `store=`, the path is a path within the repository.
+   vxf = vx.open("data/train.vortex", store=store)
+
+Listing
+=======
+
+The Hub does not implement WebDAV ``PROPFIND``, which is how object-store HTTP listing works, so a
+Hub store cannot list a prefix. Opening a known path works, since that is a ``HEAD`` plus ranged
+``GET``. To expand a glob, list the repository through the Hub's own API first — which is what
+``vortex.datasets.load_dataset`` does — and then open each path it returns.
+
+Hugging Face Datasets
+=====================
+
+``vortex.datasets.load_dataset`` builds on this to load Vortex files from the Hub as Hugging Face
+``Datasets`` objects, expanding globs and pushing projections, filters and row limits into each
+scan. See :doc:`../datasets`.

docs/api/python/store/huggingface.rst

@@ -6,8 +6,8 @@ Vortex can read from and write to Tencent Cloud COS, Alibaba Cloud OSS, and Tenc
 GooseFS through `OpenDAL <https://opendal.apache.org/>`_, which provides native service
 support.
 
-These stores are available only when Vortex is built with the ``opendal`` feature
-(e.g. ``maturin develop --features opendal`` or ``cargo build -p vortex-jni --features opendal``).
+The Python bindings always include these stores. Other consumers opt in with the ``opendal``
+Cargo feature (e.g. ``cargo build -p vortex-jni --features opendal``).
 
 .. list-table::
    :header-rows: 1
@@ -39,9 +39,6 @@ These stores are available only when Vortex is built with the ``opendal`` featur
    :func:`vortex.io.read_url` / :func:`vortex.io.write` via the ``store=`` argument,
    exactly like the built-in S3/Azure/GCS stores.
 
-   The class is only available when Vortex is built with the ``opendal`` feature; on
-   a default build, instantiating it raises :class:`ImportError`.
-
    :param bucket: COS bucket name (e.g. ``"my-bucket"``).
    :param endpoint: COS endpoint (e.g. ``"https://cos.ap-guangzhou.myqcloud.com"``).
    :param secret_id: Optional Tencent Cloud secret id. Maps to the ``TENCENTCLOUD_SECRET_ID``
@@ -62,9 +59,6 @@ These stores are available only when Vortex is built with the ``opendal`` featur
    :func:`vortex.io.read_url` / :func:`vortex.io.write` via the ``store=`` argument,
    exactly like the built-in S3/Azure/GCS stores.
 
-   The class is only available when Vortex is built with the ``opendal`` feature; on
-   a default build, instantiating it raises :class:`ImportError`.
-
    :param master_addr: GooseFS master address(es). Single master:
        ``"10.0.0.1:9200"``. HA (comma-separated):
        ``"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"``.

docs/api/python/store/opendal.rst

@@ -57,6 +57,13 @@
     # classes are fully documented in `opendal.rst`; the private paths are intentionally not.
     ("py:class", "vortex.store._cos.CosStore"),
     ("py:class", "vortex.store._goosefs.GoosefsStore"),
+    # `vortex.store.CosStore` / `GoosefsStore` / `HfStore` are the native classes re-exported
+    # through private modules, so annotations resolve to their `vortex._lib` module paths. The
+    # public classes are fully documented in `opendal.rst` / `huggingface.rst`; the native paths
+    # are intentionally not.
+    ("py:class", "vortex._lib.CosStore"),
+    ("py:class", "vortex._lib.GoosefsStore"),
+    ("py:class", "vortex._lib.HfStore"),
 ]
 
 doctest_global_setup = "import pyarrow; import vortex; import vortex as vx; import random; random.seed(a=0)"

docs/conf.py

@@ -17,14 +17,20 @@ categories = { workspace = true }
 all-features = true
 
 [dependencies]
+http = { workspace = true, optional = true }
 object_store = { workspace = true, features = ["fs"] }
 object_store_opendal = { workspace = true, optional = true }
 opendal = { workspace = true, optional = true }
 parking_lot = { workspace = true, optional = true }
+percent-encoding = { workspace = true, optional = true }
 tracing = { workspace = true, optional = true }
 url = { workspace = true }
 vortex-utils = { workspace = true }
 
+[dev-dependencies]
+rstest = { workspace = true }
+tempfile = { workspace = true }
+
 [features]
 default = []
 # The URL -> ObjectStore registry, plus the cloud backends it resolves URLs to. Kept optional so
@@ -36,6 +42,9 @@ registry = [
     "object_store/gcp",
     "object_store/http",
 ]
+# The Hugging Face Hub, the `hf://` scheme. Served over `object_store`'s HTTP store, so it adds no
+# cloud SDK of its own.
+hf = ["dep:http", "dep:percent-encoding", "object_store/http"]
 # Tencent Cloud COS, the `cos://` scheme.
 cos = [
     "dep:opendal",