feat: Bots work as coworkers

.github/workflows/ci.yml

@@ -87,6 +87,23 @@ jobs:
       - run: bun run typecheck
         working-directory: ${{ matrix.package }}
 
+  mobile:
+    name: native types
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: mobile/package-lock.json
+      # The native app is an isolated npm package; adding it to Bun workspaces would change the
+      # existing desktop/server installation contract. CI installs its own reviewed lockfile.
+      - run: npm ci --prefix mobile --ignore-scripts
+      - run: npm run --prefix mobile typecheck
+
   chart:
     name: chart (${{ matrix.target }})
     runs-on: ubuntu-latest
@@ -565,7 +582,7 @@ jobs:
     name: verify
     runs-on: ubuntu-latest
     if: always()
-    needs: [static, deployables, chart, test, startup, python-harness, build, migrations, image, component-dockerfiles]
+    needs: [static, deployables, mobile, chart, test, startup, python-harness, build, migrations, image, component-dockerfiles]
     steps:
       - name: Require every check
         env:

CHANGELOG.md

@@ -8,6 +8,27 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### Bots work as coworkers
+
+A Bot can now carry on without anyone watching it. It runs standing **Responsibilities** fed by
+schedules, signed webhooks, GitHub, Linear, Sentry, PagerDuty, inbound email and Slack messages,
+drives its own computer while the app is closed, and keeps its browser profile, cookies and files
+across restarts. Its questions and approval requests reach the person who owns the conversation in
+Slack or Microsoft Teams (through OpenTag), by text message or by push, and the conversation
+resumes when they answer; charts reach Slack and Teams as native charts. **Reachability** links
+each of those places to a conversation.
+
+Approvals become one personal flow across the browser, connected apps, shell, files, the host and
+remote Bots, with custom rules, auto-review and host command modes (**Approvals**). Bots can
+message each other, share a group conversation with attributed speakers, and be published to
+teammates as **Team Bots**. **Memory** imports facts from connected apps with their source, and
+optional background research suggests next steps. A browser demonstration can be recorded and
+turned into a skill. Administrators get capability toggles, SSO-required sign-in, SCIM, network
+egress policy, action recording, OpenTelemetry export, and **Passwords** with private sign-in
+requests.
+
+Upgrading runs migrations `0048_coworker_parity` and `0049_coworker_parity_lanes`.
+
 ### `start.sh` names the port to change on macOS
 
 When the API server's or the app's port was held by another process, `start.sh` was meant to say

agent-computer/src/authorisation.ts

@@ -73,6 +73,8 @@ const ACTING_PATHS = new Set([
   "/scroll",
   "/exec",
   "/files/write",
+  // Typing a person's login into the page. Refused while they hold the wheel, like any other typing.
+  "/sign-in/fill",
 ]);
 
 export function actsOnTheComputer(pathname: string): boolean {

agent-computer/src/control.ts

@@ -165,14 +165,12 @@ export function createControl(
   }
   function interrupt(reason: string): ControlState {
     const request = current();
+    // Recovery is for a page a person was driving: what they left it as is unknown until someone
+    // looks. A request nobody took left the page exactly as the Bot had it, and holding the Bot
+    // until a person "checks" a page no person touched stranded it asking for control it had.
+    const personHadIt = data.holder === "human" || request?.status === "taken";
     if (request && active(request)) finish(request, "interrupted", reason);
-    if (
-      request &&
-      (active(request) ||
-        request.status === "interrupted" ||
-        data.holder === "human")
-    )
-      data.recoveryRequired = true;
+    if (personHadIt) data.recoveryRequired = true;
     data.holder = "bot";
     data.since = now();
     data.resumeSnapshotRequired = true;

agent-computer/src/demonstration.ts

@@ -0,0 +1,142 @@
+import type { Page } from "playwright";
+import type { InputMessage } from "./screencast";
+export type RecordedGesture = {
+  kind: "click" | "type" | "key" | "scroll";
+  key?: string;
+  deltaY?: number;
+};
+export function gestureForRecording(
+  input: InputMessage,
+): RecordedGesture | null {
+  if (input.type === "mouse")
+    return input.event === "released" && input.button !== "right"
+      ? { kind: "click" }
+      : null;
+  if (input.type === "wheel")
+    return {
+      kind: "scroll",
+      deltaY: Math.max(-2000, Math.min(2000, input.deltaY)),
+    };
+  if (input.type === "text") return { kind: "type" };
+  if (input.event !== "down") return null;
+  if (input.key.length === 1 || (input.text && input.key !== "Enter"))
+    return { kind: "type" };
+  if (
+    [
+      "Enter",
+      "Tab",
+      "Escape",
+      "Backspace",
+      "Delete",
+      "ArrowDown",
+      "ArrowUp",
+      "ArrowLeft",
+      "ArrowRight",
+      "Home",
+      "End",
+      "PageDown",
+      "PageUp",
+    ].includes(input.key)
+  )
+    return { kind: "key", key: input.key };
+  return null;
+}
+export function safeDemonstrationUrl(raw: string): string {
+  try {
+    const url = new URL(raw);
+    if (!["http:", "https:"].includes(url.protocol)) return "about:blank";
+    const path = url.pathname
+      .split("/")
+      .map((part) =>
+        part.length > 40 || /token|secret|password|credential/i.test(part)
+          ? "[redacted]"
+          : part,
+      )
+      .join("/");
+    return `${url.origin}${path}`;
+  } catch {
+    return "about:blank";
+  }
+}
+/** Never reads a field's value, page body, clipboard or image. All text entry is parameterized. */
+export async function describeHumanGesture(page: Page, input: InputMessage) {
+  const gesture = gestureForRecording(input);
+  if (!gesture) return null;
+  const point =
+    input.type === "mouse" || input.type === "wheel"
+      ? { x: input.x, y: input.y }
+      : null;
+  const target = await page.evaluate((point) => {
+    const hit = point
+      ? document.elementFromPoint(point.x, point.y)
+      : document.activeElement;
+    const closest =
+      hit?.closest(
+        "button,a,input,textarea,select,label,[role],[contenteditable=true]",
+      ) ?? hit;
+    // A click on a label's text is a click on the control it names: "Small", a radio, not a label.
+    const element =
+      closest instanceof HTMLLabelElement
+        ? (closest.control ?? closest)
+        : closest;
+    if (!element) return { role: "page", name: "Page", sensitive: false };
+    const tag = element.tagName.toLowerCase();
+    const inputType =
+      element instanceof HTMLInputElement ? element.type.toLowerCase() : "";
+    const role =
+      element.getAttribute("role") ||
+      (
+        {
+          checkbox: "checkbox",
+          radio: "radio",
+          button: "button",
+          submit: "button",
+          reset: "button",
+          image: "button",
+          range: "slider",
+          file: "button",
+        } as Record<string, string>
+      )[inputType] ||
+      (
+        {
+          button: "button",
+          a: "link",
+          input: "textbox",
+          textarea: "textbox",
+          select: "combobox",
+        } as Record<string, string>
+      )[tag] ||
+      tag;
+    const label =
+      element.getAttribute("aria-label") ||
+      (element instanceof HTMLInputElement ||
+      element instanceof HTMLTextAreaElement ||
+      element instanceof HTMLSelectElement
+        ? Array.from(element.labels ?? [])
+            .map((entry) => entry.textContent ?? "")
+            .join(" ")
+        : "") ||
+      (tag === "button" || tag === "a" ? element.textContent : "") ||
+      element.getAttribute("placeholder") ||
+      role;
+    const sensitive =
+      element.getAttribute("type") === "password" ||
+      // Short words only as whole words: "pin" is in "topping" and "shipping", "otp" in "hotpot".
+      /password|secret|token|one.time|verification|credit.card|security.code|passcode|(?:^|[^a-z])(?:cc-|(?:cvc|cvv|pin|otp)(?![a-z]))/i.test(
+        [
+          label,
+          element.getAttribute("name"),
+          element.getAttribute("autocomplete"),
+        ].join(" "),
+      );
+    const name = sensitive
+      ? "[sensitive field]"
+      : label
+          .replace(/[A-Za-z0-9_+/=-]{32,}/g, "[redacted]")
+          .replace(/[^\s@]+@[^\s@]+/g, "[redacted]")
+          .trim()
+          .slice(0, 120);
+    return { role: role.slice(0, 40), name: name || role, sensitive };
+  }, point);
+  return { ...gesture, url: safeDemonstrationUrl(page.url()), target };
+}