Browser surface: live CDP screencast of the conversation's handed-off browser

Author: MrgSub

Cargo.lock

@@ -78,6 +78,8 @@ libc = "0.2"
 gethostname = "1"
 tokio-tungstenite = { version = "0.24", features = ["rustls-tls-webpki-roots"] }
 futures = "0.3"
+# JPEG decode for the browser screencast; the same crate gpui already builds.
+image = { version = "0.25", default-features = false, features = ["jpeg"] }
 async-trait = "0.1"
 
 # serialization

Cargo.toml

@@ -78,6 +78,7 @@ pub enum ConversationAction {
     Messages,
     Steer,
     Terminal,
+    Browser,
     /// Ends an inter-agent (`agent:`) thread — not a contact block.
     End,
 }
@@ -91,6 +92,7 @@ impl ConversationAction {
             Self::Messages => "messages",
             Self::Steer => "steer",
             Self::Terminal => "terminal",
+            Self::Browser => "browser",
             Self::End => "end",
         }
     }
@@ -103,6 +105,19 @@ pub struct TerminalSize {
     pub rows: u16,
 }
 
+/// A short-lived capability to watch and drive the browser the conversation's
+/// agent handed off: `url` is the runner's screencast WebSocket, `token` the
+/// first message's credential. The bearer secrets stay on the platform.
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BrowserScreencast {
+    pub session_id: String,
+    /// Unix millis.
+    pub expires_at: u64,
+    pub url: String,
+    pub token: String,
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("the OAuth callback was invalid")]
@@ -1147,6 +1162,39 @@ impl Client {
     }
 
     /// `/conversations/:identity/terminal[/:terminal_id/:tail]?agentId=`.
+    /// Whether the conversation's agent currently has a browser handed off.
+    pub async fn browser_available(
+        &self,
+        access_token: &str,
+        locator: &ConversationLocator,
+    ) -> Result<bool, Error> {
+        #[derive(Deserialize)]
+        struct Availability {
+            available: bool,
+        }
+        let endpoint = self.conversation_endpoint(locator, Some(ConversationAction::Browser))?;
+        let response: Availability = self
+            .send_json(self.http.get(endpoint).bearer_auth(access_token))
+            .await?;
+        Ok(response.available)
+    }
+
+    /// Mint a screencast capability for the conversation's live browser.
+    pub async fn browser_screencast(
+        &self,
+        access_token: &str,
+        locator: &ConversationLocator,
+    ) -> Result<BrowserScreencast, Error> {
+        let mut endpoint =
+            self.conversation_endpoint(locator, Some(ConversationAction::Browser))?;
+        endpoint
+            .path_segments_mut()
+            .map_err(|_| Error::InvalidContract)?
+            .push("screencast");
+        self.send_json(self.http.post(endpoint).bearer_auth(access_token))
+            .await
+    }
+
     fn terminal_endpoint(
         &self,
         locator: &ConversationLocator,

crates/keiki-api/src/lib.rs

@@ -23,6 +23,8 @@ gpui_platform.workspace = true
 gpui_tokio.workspace = true
 tokio.workspace = true
 futures.workspace = true
+tokio-tungstenite.workspace = true
+image.workspace = true
 reqwest.workspace = true
 async-trait.workspace = true
 serde.workspace = true