microsoft
diff --git a/‎.cspell.json‎
Lines changed: 3 additions & 0 deletions b/‎.cspell.json‎
Lines changed: 3 additions & 0 deletions
diff --git a/‎.devcontainer/Dockerfile‎
Lines changed: 25 additions & 3 deletions b/‎.devcontainer/Dockerfile‎
Lines changed: 25 additions & 3 deletions
diff --git a/‎.devcontainer/devcontainer.json‎
Lines changed: 19 additions & 3 deletions b/‎.devcontainer/devcontainer.json‎
Lines changed: 19 additions & 3 deletions
diff --git a/‎.gitignore‎
Lines changed: 2 additions & 0 deletions b/‎.gitignore‎
Lines changed: 2 additions & 0 deletions
diff --git a/‎data-management/viewer/backend/src/api/services/dataset_service/lerobot_handler.py‎
Lines changed: 31 additions & 1 deletion b/‎data-management/viewer/backend/src/api/services/dataset_service/lerobot_handler.py‎
Lines changed: 31 additions & 1 deletion
diff --git a/‎data-management/viewer/backend/src/api/storage/blob_dataset.py‎
Lines changed: 17 additions & 3 deletions b/‎data-management/viewer/backend/src/api/storage/blob_dataset.py‎
Lines changed: 17 additions & 3 deletions
diff --git a/‎data-management/viewer/backend/tests/api/test_annotations.py‎
Lines changed: 3 additions & 3 deletions b/‎data-management/viewer/backend/tests/api/test_annotations.py‎
Lines changed: 3 additions & 3 deletions
diff --git a/‎data-management/viewer/backend/tests/api/test_auth.py‎
Lines changed: 16 additions & 22 deletions b/‎data-management/viewer/backend/tests/api/test_auth.py‎
Lines changed: 16 additions & 22 deletions
diff --git a/‎data-management/viewer/backend/tests/storage/test_blob_dataset.py‎
Lines changed: 21 additions & 2 deletions b/‎data-management/viewer/backend/tests/storage/test_blob_dataset.py‎
Lines changed: 21 additions & 2 deletions
@@ -25,6 +25,7 @@
 		"headlessly",
 		"hfpins",
 		"imwrite",
+		"isfolder",
 		"leisaac",
 		"minimise",
 		"misattributed",
@@ -33,6 +34,7 @@
 		"mycontainer",
 		"noout",
 		"noseparator",
+		"nvmap",
 		"pixelformat",
 		"plottable",
 		"pollable",
@@ -52,6 +54,7 @@
 		"serialised",
 		"serialises",
 		"sparkline",
+		"tegra",
 		"trackio",
 		"unbuilt",
 		"unimportable",
 
@@ -1,5 +1,25 @@
-ARG BASE_IMAGE=
-FROM ${BASE_IMAGE:-mcr.microsoft.com/devcontainers/python:3.12-bookworm}
+FROM nvidia/cuda:13.0.3-cudnn-runtime-ubuntu24.04@sha256:14f6d08d1cd4a96effbfe3101d0b56326f552c199d05e4979ee0bd616df5811b AS cuda-runtime
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends cuda-cupti-13-0=13.0.85-1 && \
+	mkdir /opt/nvpl && \
+	if [ "$(dpkg --print-architecture)" = "arm64" ]; then \
+		apt-get install -y --no-install-recommends libcudss0-cuda-13=0.8.0.10-1 libnvpl-blas0=0.6.0-1 libnvpl-lapack0=0.4.0.1-1 && \
+		cp -a /usr/lib/*-linux-gnu/libcudss/13/libcudss*.so* /usr/lib/*-linux-gnu/libnvpl*.so* /opt/nvpl/; \
+	fi && \
+	rm -rf /var/lib/apt/lists/*
+
+FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04@sha256:d94c97dd9cacf183d0a6fd12a8e87b526e9e928307674ae9c94139139c0c6eae
+
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends python3 python3-venv python-is-python3 && \
+	rm -rf /var/lib/apt/lists/*
+
+COPY --from=cuda-runtime /usr/local/cuda-13.0 /usr/local/cuda-13.0
+COPY --from=cuda-runtime /usr/lib/*-linux-gnu/libcudnn*.so* /usr/local/cuda-13.0/lib64/
+COPY --from=cuda-runtime /opt/nvpl/ /usr/local/cuda-13.0/lib64/
+RUN ln -s /usr/local/cuda-13.0 /usr/local/cuda
+ENV PATH="/usr/local/cuda/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64"
 
 ARG NPM_CONFIG_REGISTRY=
 ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://registry.npmjs.org/}
@@ -8,4 +28,6 @@ ARG PIP_INDEX_URL=
 ENV PIP_INDEX_URL=${PIP_INDEX_URL:-https://pypi.org/simple/}
 
 ARG UV_DEFAULT_INDEX=
-ENV UV_DEFAULT_INDEX=${UV_DEFAULT_INDEX:-https://pypi.org/simple/}
+# No trailing slash: uv records this URL verbatim in uv.lock, so a slash here
+# makes every locally regenerated lock drift from the CI-generated one.
+ENV UV_DEFAULT_INDEX=${UV_DEFAULT_INDEX:-https://pypi.org/simple}
@@ -6,7 +6,6 @@
 	"build": {
 		"dockerfile": "Dockerfile",
 		"args": {
-			"BASE_IMAGE": "${localEnv:PHYSICAL_AI_DEVCONTAINER_IMAGE}",
 			"NPM_CONFIG_REGISTRY": "${localEnv:NPM_CONFIG_REGISTRY}",
 			"PIP_INDEX_URL": "${localEnv:PIP_INDEX_URL}",
 			"UV_DEFAULT_INDEX": "${localEnv:UV_DEFAULT_INDEX}"
@@ -92,10 +91,27 @@
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [8080],
 
+	// GPU device nodes are group-restricted: /dev/nvmap is root:video and
+	// /dev/dri/renderD* is root:render (GID 993 here). CUDA initialization fails
+	// with error 801 unless the container user holds both; adjust 993 if the host
+	// render group differs. "gpu": "optional" attaches devices when present and
+	// still builds on GPU-less hosts.
+	"runArgs": [
+		"--group-add",
+		"video",
+		"--group-add",
+		"993"
+	],
+	"hostRequirements": {
+		"gpu": "optional"
+	},
+
 	// Container environment variables
 	"containerEnv": {
 		"BROWSER": "code --new-window",
-		"UV_LINK_MODE": "copy"
+		"UV_LINK_MODE": "copy",
+		"NVIDIA_VISIBLE_DEVICES": "all",
+		"NVIDIA_DRIVER_CAPABILITIES": "all"
 	},
 
 	"onCreateCommand": {
@@ -106,7 +122,7 @@
 		"actionlint": "./scripts/setup/install-actionlint.sh",
 		"golangci-lint": "GOLANGCI_LINT_VERSION=2.12.2 && case \"$(uname -m)\" in x86_64) GOLANGCI_LINT_ARCH=amd64; GOLANGCI_LINT_SHA256=8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553 ;; aarch64|arm64) GOLANGCI_LINT_ARCH=arm64; GOLANGCI_LINT_SHA256=44cd40a8c76c86755375adfeea52cfd3533cb43d7bd647771e0ae065e166df3a ;; *) echo \"Unsupported arch: $(uname -m)\" >&2; exit 1 ;; esac && curl -fsSL \"https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${GOLANGCI_LINT_ARCH}.tar.gz\" -o /tmp/golangci-lint.tar.gz && echo \"${GOLANGCI_LINT_SHA256}  /tmp/golangci-lint.tar.gz\" | sha256sum -c --quiet - && sudo tar -xzf /tmp/golangci-lint.tar.gz -C /usr/local/bin --strip-components=1 \"golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${GOLANGCI_LINT_ARCH}/golangci-lint\" && rm /tmp/golangci-lint.tar.gz",
 		"update-bashrc": "echo 'export PATH=\"${containerWorkspaceFolder}/scripts:${containerWorkspaceFolder}/node_modules/.bin:$PATH\"' | sudo tee -a  ~/.bashrc",
-		"osmo-cli": "if [ \"$(uname -m)\" != \"x86_64\" ]; then echo \"Skipping osmo-cli: no $(uname -m) build available (x86_64 only)\"; exit 0; fi && OSMO_VERSION=6.3.0 && OSMO_INSTALLER_SHA256=e2e70b4868fe2ee22c7952f2d53d1e013070168b77e89f51e8c52a0eaaf18928 && curl -fsSL \"https://github.com/NVIDIA/OSMO/releases/download/${OSMO_VERSION}/osmo-client-installer-${OSMO_VERSION}-linux-x86_64.sh\" -o /tmp/osmo_install.sh && echo \"${OSMO_INSTALLER_SHA256}  /tmp/osmo_install.sh\" | sha256sum -c --quiet - && sudo bash /tmp/osmo_install.sh && rm /tmp/osmo_install.sh",
+		"osmo-cli": "OSMO_VERSION=6.3.0 && case \"$(uname -m)\" in x86_64) OSMO_ARCH=x86_64; OSMO_INSTALLER_SHA256=e2e70b4868fe2ee22c7952f2d53d1e013070168b77e89f51e8c52a0eaaf18928 ;; aarch64|arm64) OSMO_ARCH=arm64; OSMO_INSTALLER_SHA256=03d559d8a999b4d6f2ac6aac989d17792a47aaee70b5c7fc5787655f123923ee ;; *) echo \"Unsupported arch: $(uname -m)\" >&2; exit 1 ;; esac && curl -fsSL \"https://github.com/NVIDIA/OSMO/releases/download/${OSMO_VERSION}/osmo-client-installer-${OSMO_VERSION}-linux-${OSMO_ARCH}.sh\" -o /tmp/osmo_install.sh && echo \"${OSMO_INSTALLER_SHA256}  /tmp/osmo_install.sh\" | sha256sum -c --quiet - && sudo bash /tmp/osmo_install.sh && rm /tmp/osmo_install.sh",
 		"ngc-cli": "if [ \"$(uname -m)\" != \"x86_64\" ]; then echo \"Skipping ngc-cli: no $(uname -m) build available (x86_64 only)\"; exit 0; fi && NGC_CLI_VERSION=3.41.4 && NGC_CLI_SHA256=2c86681048ab8e2980bdd6aa6c17f086eff988276ad90a28f1307b69fdb50252 && curl -fsSL \"https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGC_CLI_VERSION}/files/ngccli_linux.zip\" -o /tmp/ngccli.zip && echo \"${NGC_CLI_SHA256}  /tmp/ngccli.zip\" | sha256sum -c --quiet - && sudo unzip -o /tmp/ngccli.zip -d /usr/local && rm /tmp/ngccli.zip && sudo chmod u+x /usr/local/ngc-cli/ngc && sudo ln -sf /usr/local/ngc-cli/ngc /usr/local/bin/ngc",
 		"az-ml": "az extension add -n ml --yes || true"
 	},
 
@@ -75,6 +75,7 @@ project.lock.json
 project.fragment.lock.json
 artifacts/
 **/.artifacts/
+/outputs/
 
 # ASP.NET Scaffolding
 ScaffoldingReadMe.txt
@@ -344,6 +345,7 @@ paket-files/
 
 # Python Tools for Visual Studio (PTVS)
 **/__pycache__/
+**/*.egg-info/
 *.pyc
 
 # Hypothesis testing framework cache
 
@@ -475,8 +475,9 @@ def get_video_path(self, dataset_id: str, episode_idx: int, camera: str) -> str
             if clip_path is None:
                 return str(video_path)
 
-            if clip_path.exists():
+            if clip_path.exists() and self._is_valid_video_file(clip_path):
                 return str(clip_path)
+            clip_path.unlink(missing_ok=True)
 
             if self._generate_episode_video_clip(video_path, window, clip_path):
                 return str(clip_path)
@@ -495,6 +496,35 @@ def _video_cache_path(self, dataset_id: str, episode_idx: int, camera: str) -> P
         safe_camera = camera.replace("/", "_").replace("\\", "_")
         return loader.base_path / "meta" / "videos" / safe_camera / f"episode_{episode_idx:06d}.mp4"
 
+    @staticmethod
+    def _is_valid_video_file(video_path: Path) -> bool:
+        ffmpeg = LeRobotFormatHandler._resolve_ffmpeg()
+        if ffmpeg is None:
+            return video_path.stat().st_size > 0
+
+        try:
+            proc = subprocess.run(
+                [
+                    ffmpeg,
+                    "-v",
+                    "error",
+                    "-i",
+                    str(video_path),
+                    "-map",
+                    "0:v:0",
+                    "-frames:v",
+                    "1",
+                    "-f",
+                    "null",
+                    "-",
+                ],
+                capture_output=True,
+                timeout=10,
+            )
+            return proc.returncode == 0
+        except (OSError, subprocess.SubprocessError):
+            return False
+
     @staticmethod
     def _generate_episode_video_clip(source_path: Path, window: tuple[float, float], clip_path: Path) -> bool:
         ffmpeg = LeRobotFormatHandler._resolve_ffmpeg()
 
@@ -606,15 +606,20 @@ async def sync_dataset_to_local(self, dataset_id: str, local_dir: Path) -> bool:
             prefix = f"{self.get_blob_prefix(dataset_id)}/"
             synced_count = 0
 
-            async for blob in container.list_blobs(name_starts_with=prefix):
+            async for blob in container.list_blobs(name_starts_with=prefix, include=["metadata"]):
+                if self._is_directory_blob(blob):
+                    continue
+
+                relative = blob.name[len(prefix) :]
+                if relative == ".cache" or relative.startswith(".cache/"):
+                    continue
                 # Skip video files — they are streamed on demand
                 if "/videos/" in blob.name:
                     continue
                 # Skip HDF5 files — they are downloaded on demand per episode
                 if blob.name.endswith(".hdf5"):
                     continue
 
-                relative = blob.name[len(prefix) :]
                 local_path = local_dir / relative
                 local_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -642,6 +647,12 @@ async def sync_dataset_to_local(self, dataset_id: str, local_dir: Path) -> bool:
             )
             return False
 
+    @staticmethod
+    def _is_directory_blob(blob: object) -> bool:
+        """Return whether a listed blob is an ADLS Gen2 directory resource."""
+        metadata = getattr(blob, "metadata", None)
+        return isinstance(metadata, dict) and str(metadata.get("hdi_isfolder", "")).lower() == "true"
+
     async def sync_meta_only_to_local(self, dataset_id: str, local_dir: Path) -> bool:
         """
         Download only meta/ files for a dataset to a local directory.
@@ -665,7 +676,10 @@ async def sync_meta_only_to_local(self, dataset_id: str, local_dir: Path) -> boo
             prefix = self.get_blob_prefix(dataset_id)
             meta_prefix = f"{prefix}/meta/"
 
-            async for blob in container.list_blobs(name_starts_with=meta_prefix):
+            async for blob in container.list_blobs(name_starts_with=meta_prefix, include=["metadata"]):
+                if self._is_directory_blob(blob):
+                    continue
+
                 relative = blob.name[len(f"{prefix}/") :]
                 if relative not in _SYNC_META_BLOBS and not relative.startswith("meta/episodes/"):
                     continue
 
@@ -3,7 +3,6 @@
 """
 
 import asyncio
-import os
 import tempfile
 from datetime import UTC, datetime
 
@@ -29,10 +28,11 @@
 
 
 @pytest.fixture
-def client():
+def client(monkeypatch):
     """Create test client with isolated singletons and empty temp data path."""
     with tempfile.TemporaryDirectory() as tmp:
-        os.environ["DATA_DIR"] = tmp
+        monkeypatch.setenv("DATA_DIR", tmp)
+        monkeypatch.setenv("STORAGE_BACKEND", "local")
 
         import src.api.config as config_mod
         import src.api.services.annotation_service as ann_mod
 
@@ -1,6 +1,6 @@
 """Unit tests for authentication and CSRF middleware."""
 
-import os
+from __future__ import annotations
 
 import pytest
 from fastapi.testclient import TestClient
@@ -9,49 +9,43 @@
 
 
 @pytest.fixture(autouse=True)
-def reset_auth_state(tmp_path):
-    """Reset the auth provider singleton and set a valid DATA_DIR before each test."""
+def reset_auth_state(tmp_path, monkeypatch):
+    """Reset application state and use isolated local storage for each test."""
     from src.api import auth as auth_mod
+    from src.api import config as config_mod
+    from src.api import main as main_mod
 
-    os.environ["DATA_DIR"] = str(tmp_path)
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    monkeypatch.setenv("STORAGE_BACKEND", "local")
+    config_mod._app_config = None
+    monkeypatch.setattr(main_mod, "_config", config_mod.load_config())
     auth_mod.reset_auth_provider()
     yield
     auth_mod.reset_auth_provider()
-    os.environ.pop("DATA_DIR", None)
+    config_mod._app_config = None
 
 
 @pytest.fixture
-def client_with_auth():
+def client_with_auth(monkeypatch):
     """Test client with authentication enabled and API-key provider configured."""
     import src.api.services.dataset_service as ds_mod
 
     ds_mod._dataset_service = None
-    original_disabled = os.environ.pop("DATAVIEWER_AUTH_DISABLED", None)
-    os.environ["DATAVIEWER_AUTH_PROVIDER"] = "apikey"
-    os.environ["DATAVIEWER_API_KEY"] = "test-secret-key"
+    monkeypatch.delenv("DATAVIEWER_AUTH_DISABLED", raising=False)
+    monkeypatch.setenv("DATAVIEWER_AUTH_PROVIDER", "apikey")
+    monkeypatch.setenv("DATAVIEWER_API_KEY", "test-secret-key")
     yield TestClient(app)
-    if original_disabled is not None:
-        os.environ["DATAVIEWER_AUTH_DISABLED"] = original_disabled
-    else:
-        os.environ.pop("DATAVIEWER_AUTH_DISABLED", None)
-    os.environ.pop("DATAVIEWER_AUTH_PROVIDER", None)
-    os.environ.pop("DATAVIEWER_API_KEY", None)
     ds_mod._dataset_service = None
 
 
 @pytest.fixture
-def client_auth_disabled():
+def client_auth_disabled(monkeypatch):
     """Test client with auth disabled (DATAVIEWER_AUTH_DISABLED=true)."""
     import src.api.services.dataset_service as ds_mod
 
     ds_mod._dataset_service = None
-    original = os.environ.get("DATAVIEWER_AUTH_DISABLED")
-    os.environ["DATAVIEWER_AUTH_DISABLED"] = "true"
+    monkeypatch.setenv("DATAVIEWER_AUTH_DISABLED", "true")
     yield TestClient(app)
-    if original is not None:
-        os.environ["DATAVIEWER_AUTH_DISABLED"] = original
-    else:
-        os.environ.pop("DATAVIEWER_AUTH_DISABLED", None)
     ds_mod._dataset_service = None
 
 
 
@@ -32,6 +32,12 @@ def _make_blob(name: str):
     return blob
 
 
+def _make_directory_blob(name: str):
+    blob = _make_blob(name)
+    blob.metadata = {"hdi_isfolder": "true"}
+    return blob
+
+
 def _build_provider(mock_client=None):
     from src.api.storage.blob_dataset import BlobDatasetProvider
 
@@ -544,7 +550,14 @@ def test_sync_dataset_skips_videos_and_hdf5(self):
             "org/repo/extra/episode_0.hdf5",  # skipped
         ]
         mock_container = MagicMock()
-        mock_container.list_blobs.return_value = _AsyncIter(_make_blob(n) for n in names)
+        mock_container.list_blobs.return_value = _AsyncIter(
+            [
+                _make_directory_blob("org/repo/data"),
+                _make_directory_blob("org/repo/data/chunk-000"),
+                _make_blob("org/repo/.cache/huggingface/download.lock"),
+                *(_make_blob(name) for name in names),
+            ]
+        )
         mock_client = MagicMock()
         mock_client.get_container_client.return_value = mock_container
         provider = _build_provider(mock_client)
@@ -559,6 +572,7 @@ def test_sync_dataset_skips_videos_and_hdf5(self):
             assert (local_dir / "meta" / "info.json").read_bytes() == b"data"
             assert (local_dir / "data" / "chunk-000" / "file-000.parquet").exists()
             assert not (local_dir / "videos").exists()
+            assert not (local_dir / ".cache").exists()
             assert read_mock.await_count == 2
 
     @patch("src.api.storage.blob_dataset.AZURE_AVAILABLE", True)
@@ -584,7 +598,12 @@ def test_sync_meta_only_filters_to_allowed_blobs(self):
             "org/repo/meta/something_else.json",  # filtered out
         ]
         mock_container = MagicMock()
-        mock_container.list_blobs.return_value = _AsyncIter(_make_blob(n) for n in names)
+        mock_container.list_blobs.return_value = _AsyncIter(
+            [
+                _make_directory_blob("org/repo/meta/episodes"),
+                *(_make_blob(name) for name in names),
+            ]
+        )
         mock_client = MagicMock()
         mock_client.get_container_client.return_value = mock_container
         provider = _build_provider(mock_client)