Merge pull request #2039 from unclecode/fix/docker-ui-auth-gate

fix(docker): let dashboard/playground load when auth gate is active

Author: ntohidi

deploy/docker/auth_gate.py

@@ -15,6 +15,7 @@
   * a valid HS256 JWT minted by this server -> the token's own scope claim.
 
 Public paths (the health check and the token-issuing endpoint) pass through.
+Public prefixes (the UI static shells) also pass through - they serve no data.
 On failure: HTTP 401 JSON, or WebSocket close 4401.
 On success: the validated principal is attached at scope["state"]["principal"]
 (readable downstream as request.state.principal) for scope/ownership checks.
@@ -38,18 +39,24 @@ def __init__(
         *,
         token_provider: Callable[[], str],
         public_paths: Iterable[str] = (),
+        public_prefixes: Iterable[str] = (),
     ):
         self.app = app
         self._token_provider = token_provider
         self.public_paths = set(public_paths)
+        self.public_prefixes = tuple(public_prefixes)
 
     # ─────────────────────────── ASGI entry ───────────────────────────
     async def __call__(self, scope, receive, send):
         if scope["type"] not in ("http", "websocket"):
             await self.app(scope, receive, send)
             return
 
-        if scope.get("path", "") in self.public_paths:
+        path = scope.get("path", "")
+        if path in self.public_paths:
+            await self.app(scope, receive, send)
+            return
+        if self.public_prefixes and path.startswith(self.public_prefixes):
             await self.app(scope, receive, send)
             return
 

deploy/docker/server.py

@@ -393,6 +393,7 @@ def _current_api_token() -> str:
     AuthGateMiddleware,
     token_provider=_current_api_token,
     public_paths={HEALTH_PATH, "/token"},
+    public_prefixes=_UI_PREFIXES,
 )
 
 # ── request body-size limit (DoS) ─────────────────────────────────────

deploy/docker/static/monitor/index.html

@@ -93,6 +93,14 @@ <h1 class="text-lg font-medium flex items-center space-x-4">
         </h1>
 
         <div class="ml-auto flex items-center space-x-4">
+            <!-- Token bar -->
+            <div class="flex items-center space-x-2">
+                <input id="token-input" type="password" placeholder="API token"
+                       class="bg-dark border border-border rounded px-2 py-1 text-xs w-48">
+                <button id="token-save" class="px-2 py-1 bg-primary text-dark rounded text-xs hover:bg-primarydim">Set</button>
+                <button id="token-clear" class="px-2 py-1 border border-border rounded text-xs hover:bg-surface">Clear</button>
+                <span id="token-status" class="text-xs"></span>
+            </div>
             <!-- Connection Status -->
             <div class="flex items-center space-x-2">
                 <div id="ws-status" class="flex items-center space-x-1">
@@ -318,6 +326,47 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
     </main>
 
     <script>
+        // ========== Auth ==========
+        function getToken() { return sessionStorage.getItem('crawl4ai_token') || ''; }
+
+        function authFetch(url, opts = {}) {
+            const token = getToken();
+            if (token) {
+                opts.headers = { ...opts.headers, 'Authorization': 'Bearer ' + token };
+            }
+            return fetch(url, opts);
+        }
+
+        (function initTokenBar() {
+            const input = document.getElementById('token-input');
+            const status = document.getElementById('token-status');
+            const saved = sessionStorage.getItem('crawl4ai_token');
+            if (saved) input.value = saved;
+
+            function applyToken() {
+                const val = input.value.trim();
+                if (!val) return;
+                sessionStorage.setItem('crawl4ai_token', val);
+                status.textContent = 'Token saved';
+                status.className = 'text-xs text-green-400';
+                setTimeout(() => { status.textContent = ''; }, 2000);
+                // Reconnect WebSocket with new token
+                if (websocket) { websocket.close(); }
+                wsReconnectAttempts = 0;
+                connectWebSocket();
+            }
+
+            document.getElementById('token-save').addEventListener('click', applyToken);
+            input.addEventListener('keydown', e => { if (e.key === 'Enter') applyToken(); });
+            document.getElementById('token-clear').addEventListener('click', () => {
+                sessionStorage.removeItem('crawl4ai_token');
+                input.value = '';
+                status.textContent = 'Token cleared';
+                status.className = 'text-xs text-yellow-400';
+                setTimeout(() => { status.textContent = ''; }, 2000);
+            });
+        })();
+
         // ========== State Management ==========
         let autoRefresh = true;
         let refreshInterval;
@@ -368,7 +417,9 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
             wsReconnectAttempts++;
 
             const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
-            const wsUrl = `${protocol}//${window.location.host}/monitor/ws`;
+            let wsUrl = `${protocol}//${window.location.host}/monitor/ws`;
+            const wsToken = getToken();
+            if (wsToken) wsUrl += `?token=${encodeURIComponent(wsToken)}`;
 
             websocket = new WebSocket(wsUrl);
 
@@ -643,7 +694,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
 
         async function fetchHealth() {
             try {
-                const res = await fetch('/monitor/health');
+                const res = await authFetch('/monitor/health');
                 const data = await res.json();
 
                 // Container metrics
@@ -691,7 +742,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
         async function fetchRequests() {
             try {
                 const filter = document.getElementById('filter-requests')?.value || 'all';
-                const res = await fetch(`/monitor/requests?status=${filter}&limit=50`);
+                const res = await authFetch(`/monitor/requests?status=${filter}&limit=50`);
                 const data = await res.json();
 
                 // Active requests
@@ -735,7 +786,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
 
         async function fetchBrowsers() {
             try {
-                const res = await fetch('/monitor/browsers');
+                const res = await authFetch('/monitor/browsers');
                 const data = await res.json();
 
                 document.getElementById('browser-count').textContent = data.summary.total_count;
@@ -775,7 +826,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
 
         async function fetchJanitorLog() {
             try {
-                const res = await fetch('/monitor/logs/janitor?limit=100');
+                const res = await authFetch('/monitor/logs/janitor?limit=100');
                 const data = await res.json();
 
                 const logEl = document.getElementById('janitor-log');
@@ -803,7 +854,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
 
         async function fetchErrors() {
             try {
-                const res = await fetch('/monitor/logs/errors?limit=100');
+                const res = await authFetch('/monitor/logs/errors?limit=100');
                 const data = await res.json();
 
                 const logEl = document.getElementById('errors-log');
@@ -830,7 +881,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
 
         async function fetchEndpointStats() {
             try {
-                const res = await fetch('/monitor/endpoints/stats');
+                const res = await authFetch('/monitor/endpoints/stats');
                 const data = await res.json();
 
                 const tbody = document.getElementById('endpoints-table-body');
@@ -861,7 +912,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
         async function fetchTimeline() {
             try {
                 const metric = document.getElementById('timeline-metric').value;
-                const res = await fetch(`/monitor/timeline?metric=${metric}`);
+                const res = await authFetch(`/monitor/timeline?metric=${metric}`);
                 const data = await res.json();
 
                 drawTimeline(data, metric);
@@ -968,7 +1019,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
             if (!confirm(`Kill browser ${sig}?`)) return;
 
             try {
-                const res = await fetch('/monitor/actions/kill_browser', {
+                const res = await authFetch('/monitor/actions/kill_browser', {
                     method: 'POST',
                     headers: {'Content-Type': 'application/json'},
                     body: JSON.stringify({sig})
@@ -986,7 +1037,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
             if (!confirm(`Restart browser ${sig}?`)) return;
 
             try {
-                const res = await fetch('/monitor/actions/restart_browser', {
+                const res = await authFetch('/monitor/actions/restart_browser', {
                     method: 'POST',
                     headers: {'Content-Type': 'application/json'},
                     body: JSON.stringify({sig})
@@ -1004,7 +1055,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
             if (!confirm('Force cleanup all cold pool browsers?')) return;
 
             try {
-                const res = await fetch('/monitor/actions/cleanup', {method: 'POST'});
+                const res = await authFetch('/monitor/actions/cleanup', {method: 'POST'});
                 const data = await res.json();
 
                 showActionStatus(`✅ Killed ${data.killed_browsers} browsers`, true);
@@ -1018,7 +1069,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
             if (!confirm('Restart permanent browser? This will briefly interrupt service.')) return;
 
             try {
-                const res = await fetch('/monitor/actions/restart_browser', {
+                const res = await authFetch('/monitor/actions/restart_browser', {
                     method: 'POST',
                     headers: {'Content-Type': 'application/json'},
                     body: JSON.stringify({sig: 'permanent'})
@@ -1036,7 +1087,7 @@ <h2 class="text-sm font-medium mb-3 text-accent">Control Actions</h2>
             if (!confirm('Reset all endpoint statistics?')) return;
 
             try {
-                const res = await fetch('/monitor/stats/reset', {method: 'POST'});
+                const res = await authFetch('/monitor/stats/reset', {method: 'POST'});
                 const data = await res.json();
 
                 showActionStatus('✅ Stats reset', true);

deploy/docker/static/playground/index.html

@@ -172,6 +172,14 @@ <h1 class="text-lg font-medium flex items-center space-x-4">
         </h1>
 
         <div class="ml-auto flex items-center space-x-4">
+            <!-- Token bar -->
+            <div id="token-bar" class="flex items-center space-x-2">
+                <input id="token-input" type="password" placeholder="API token"
+                       class="bg-dark border border-border rounded px-2 py-1 text-xs w-48">
+                <button id="token-save" class="px-2 py-1 bg-primary text-dark rounded text-xs hover:bg-primarydim">Set</button>
+                <button onclick="clearToken()" class="px-2 py-1 border border-border rounded text-xs hover:bg-surface">Clear</button>
+                <span id="token-status" class="text-xs"></span>
+            </div>
             <a href="/dashboard" class="text-xs text-secondary hover:text-primary underline">Monitor</a>
             <div class="flex space-x-2">
                 <button id="play-tab"
@@ -402,6 +410,44 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
     </div>
 
     <script>
+        // ================ AUTH ================
+        const _savedToken = sessionStorage.getItem('crawl4ai_token');
+        const _tokenBar = document.getElementById('token-bar');
+        const _tokenInput = document.getElementById('token-input');
+        const _tokenSave = document.getElementById('token-save');
+        const _tokenStatus = document.getElementById('token-status');
+
+        function getToken() { return sessionStorage.getItem('crawl4ai_token') || ''; }
+
+        function authFetch(url, opts = {}) {
+            const token = getToken();
+            if (token) {
+                opts.headers = { ...opts.headers, 'Authorization': 'Bearer ' + token };
+            }
+            return fetch(url, opts);
+        }
+
+        function applyToken() {
+            const val = _tokenInput.value.trim();
+            if (!val) return;
+            sessionStorage.setItem('crawl4ai_token', val);
+            _tokenStatus.textContent = 'Token saved';
+            _tokenStatus.className = 'text-xs text-green-400';
+            setTimeout(() => { _tokenStatus.textContent = ''; }, 2000);
+        }
+
+        function clearToken() {
+            sessionStorage.removeItem('crawl4ai_token');
+            _tokenInput.value = '';
+            _tokenStatus.textContent = 'Token cleared';
+            _tokenStatus.className = 'text-xs text-yellow-400';
+            setTimeout(() => { _tokenStatus.textContent = ''; }, 2000);
+        }
+
+        if (_savedToken) _tokenInput.value = _savedToken;
+        _tokenSave.addEventListener('click', applyToken);
+        _tokenInput.addEventListener('keydown', e => { if (e.key === 'Enter') applyToken(); });
+
         // Tab switching
         document.querySelectorAll('.tab-btn').forEach(btn => {
             btn.addEventListener('click', () => {
@@ -517,7 +563,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             const code = cm.getValue().trim();
             if (!code) return {};
 
-            const res = await fetch('/config/dump', {
+            const res = await authFetch('/config/dump', {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
                 body: JSON.stringify({ code }),
@@ -691,7 +737,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                     // Get the question from the LLM-specific input
                     const question = document.getElementById('llm-question').value.trim() || "What is this page about?";
 
-                    response = await fetch(`${api}/${encodedUrl}?q=${encodeURIComponent(question)}`, {
+                    response = await authFetch(`${api}/${encodedUrl}?q=${encodeURIComponent(question)}`, {
                         method: 'GET',
                         headers: { 'Accept': 'application/json' }
                     });
@@ -707,7 +753,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                     forceHighlightElement(document.querySelector('#response-content code'));
                 } else if (endpoint === 'crawl_stream' || useStreamOverride) {
                     // Stream processing - now handled directly by /crawl endpoint
-                    response = await fetch(api, {
+                    response = await authFetch(api, {
                         method: 'POST',
                         headers: { 'Content-Type': 'application/json' },
                         body: JSON.stringify(payload)
@@ -746,7 +792,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                     forceHighlightElement(document.querySelector('#response-content code'));
                 } else {
                     // Regular request (handles /crawl and /md)
-                    response = await fetch(api, {
+                    response = await authFetch(api, {
                         method: 'POST',
                         headers: { 'Content-Type': 'application/json' },
                         body: JSON.stringify(payload)
@@ -835,7 +881,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
 
                 try {
                     if (useStream) {
-                        const response = await fetch(api, {
+                        const response = await authFetch(api, {
                             method: 'POST',
                             headers: { 'Content-Type': 'application/json' },
                             body: JSON.stringify(payload)
@@ -859,7 +905,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
 
                         memory = maxMem;
                     } else {
-                        const response = await fetch(api, {
+                        const response = await authFetch(api, {
                             method: 'POST',
                             headers: { 'Content-Type': 'application/json' },
                             body: JSON.stringify(payload)

deploy/docker/tests/test_security_default_posture.py

@@ -6,7 +6,8 @@
 asserts the secure-by-default end state:
 
   - every mutating + read endpoint, static mount, and MCP transport requires
-    auth out of the box (R1); only the health check is public,
+    auth out of the box (R1); only the health check and UI shell pages
+    (dashboard/playground) are public,
   - the token endpoint will not freely mint credentials when no api_token /
     secret is configured (R1),
   - strong security headers + a strict CSP are present on every response even
@@ -68,11 +69,12 @@
     ("get", "/monitor/timeline", None),
     # MCP transport (must be gated; today it is open AND launders credentials)
     ("get", "/mcp/schema", None),
-    # static mounts (dashboard / playground served unauthenticated today)
-    ("get", "/dashboard/", None),
-    ("get", "/playground/", None),
 ]
 
+# UI shell pages that are intentionally public — they serve only static
+# HTML/CSS/JS and contain no data.  All data routes behind them remain gated.
+PUBLIC_UI_PATHS = ["/dashboard/", "/playground/"]
+
 
 def _call(client, method, path, body):
     fn = getattr(client, method)
@@ -91,6 +93,15 @@ def test_health_check_is_public(self, stock_client):
         r = stock_client.get(HEALTH)
         assert r.status_code == 200
 
+    @pytest.mark.parametrize("path", PUBLIC_UI_PATHS, ids=PUBLIC_UI_PATHS)
+    def test_ui_shell_is_public(self, stock_client, path):
+        """UI shells serve static HTML only — no data — so they load without auth."""
+        r = stock_client.get(path)
+        assert r.status_code == 200, (
+            f"GET {path} returned {r.status_code}; expected 200. "
+            f"UI shell pages must be publicly accessible."
+        )
+
     @pytest.mark.parametrize(
         "method,path,body",
         PROTECTED_ENDPOINTS,