Merge pull request #201 from LockInTime/feat/195-response-pagination

feat(protocol): add bounded list pagination

Author: SarthakWade

apps/headless/Host/AgentBridge.swift

@@ -321,12 +321,16 @@ extension BrowserWindowController {
     func agentQAReport() -> JSONValue { qaBridge.store.report() }
     func agentQAClear() -> JSONValue { qaBridge.clear() }
 
-    func agentConsole(level: String, limit: Int) -> JSONValue {
-        qaBridge.store.console(level: level, limit: limit)
+    func agentConsole(level: String, limit: Int, cursor: String?) throws -> JSONValue {
+        try qaBridge.store.console(level: level, limit: limit, cursor: cursor)
     }
 
-    func agentNetwork(failedOnly: Bool, status: Int?, limit: Int) -> JSONValue {
-        qaBridge.store.network(failedOnly: failedOnly, status: status, limit: limit)
+    func agentNetwork(
+        failedOnly: Bool, status: Int?, limit: Int, cursor: String?
+    ) throws -> JSONValue {
+        try qaBridge.store.network(
+            failedOnly: failedOnly, status: status, limit: limit, cursor: cursor
+        )
     }
 
     func agentNetworkDetail(requestID: String) -> JSONValue {
@@ -519,6 +523,16 @@ final class WebKitBrowserEngine: BrowserEngine {
 extension BrowserWindowController: BrowserEngineSession {
     var hostIsolated: Bool { isIsolatedSession }
 
+    func hostSessionMetadata() throws -> BrowserSessionPageMetadata {
+        onMain {
+            BrowserSessionPageMetadata(
+                url: self.webView.url?.absoluteString,
+                title: self.webView.title,
+                lifecycle: self.webView.isLoading ? .navigating : .available
+            )
+        }
+    }
+
     func hostEnableAgentControl() { onMain { self.enableAgentControl() } }
     func hostVisit(_ url: URL) throws -> JSONValue { try agentVisit(url) }
     func hostInspect(parameters: [String: JSONValue]) throws -> JSONValue {
@@ -572,11 +586,11 @@ extension BrowserWindowController: BrowserEngineSession {
     }
     func hostQAReport() throws -> JSONValue { agentQAReport() }
     func hostQAClear() throws -> JSONValue { agentQAClear() }
-    func hostConsole(level: String, limit: Int) throws -> JSONValue {
-        agentConsole(level: level, limit: limit)
+    func hostConsole(level: String, limit: Int, cursor: String?) throws -> JSONValue {
+        try agentConsole(level: level, limit: limit, cursor: cursor)
     }
-    func hostNetwork(failedOnly: Bool, status: Int?, limit: Int) throws -> JSONValue {
-        agentNetwork(failedOnly: failedOnly, status: status, limit: limit)
+    func hostNetwork(failedOnly: Bool, status: Int?, limit: Int, cursor: String?) throws -> JSONValue {
+        try agentNetwork(failedOnly: failedOnly, status: status, limit: limit, cursor: cursor)
     }
     func hostNetworkDetail(requestID: String) throws -> JSONValue {
         agentNetworkDetail(requestID: requestID)

apps/headless/LinuxHost/BrowserProcess.swift

@@ -616,6 +616,7 @@ final class LinuxBrowserSession: @unchecked Sendable {
     private var mainFrameID: String?
     private var lastSafeURL: String?
     private var navigationRecoveryPending = false
+    private var navigationInProgress = false
     private var isolatedContextID: Int?
     private let mockLock = NSLock()
     private var networkMocks: [NetworkMock] = []
@@ -671,12 +672,30 @@ final class LinuxBrowserSession: @unchecked Sendable {
     }
 
     func visit(_ url: URL) throws -> JSONValue {
-        navigationLock.lock(); lastSafeURL = url.absoluteString; navigationLock.unlock()
+        navigationLock.lock()
+        lastSafeURL = url.absoluteString
+        navigationInProgress = true
+        navigationLock.unlock()
         pauseRecordingCapture()
         _ = try command("Page.navigate", parameters: ["url": url.absoluteString])
         return try wait(parameters: ["settled": .bool(true), "timeoutMs": .number(20_000)])
     }
 
+    func sessionMetadata() throws -> BrowserSessionPageMetadata {
+        let history = try command("Page.getNavigationHistory", timeoutMilliseconds: 2_000)
+        let currentIndex = (history["currentIndex"] as? NSNumber)?.intValue ?? -1
+        let entries = history["entries"] as? [[String: Any]] ?? []
+        let entry = entries.indices.contains(currentIndex) ? entries[currentIndex] : nil
+        navigationLock.lock()
+        let navigating = navigationInProgress
+        navigationLock.unlock()
+        return BrowserSessionPageMetadata(
+            url: entry?["url"] as? String,
+            title: entry?["title"] as? String,
+            lifecycle: navigating ? .navigating : .available
+        )
+    }
+
     func inspect(parameters: [String: JSONValue]) throws -> JSONValue {
         var options: [String: Any] = [
             "context": parameters["context"]?.stringValue ?? "full",
@@ -965,12 +984,16 @@ final class LinuxBrowserSession: @unchecked Sendable {
         return diagnostics.report()
     }
 
-    func console(level: String, limit: Int) -> JSONValue {
-        diagnostics.console(level: level, limit: limit)
+    func console(level: String, limit: Int, cursor: String?) throws -> JSONValue {
+        try diagnostics.console(level: level, limit: limit, cursor: cursor)
     }
 
-    func network(failedOnly: Bool, status: Int?, limit: Int) -> JSONValue {
-        diagnostics.network(failedOnly: failedOnly, status: status, limit: limit)
+    func network(
+        failedOnly: Bool, status: Int?, limit: Int, cursor: String?
+    ) throws -> JSONValue {
+        try diagnostics.network(
+            failedOnly: failedOnly, status: status, limit: limit, cursor: cursor
+        )
     }
 
     func networkDetail(requestID: String) -> JSONValue {
@@ -1106,6 +1129,22 @@ final class LinuxBrowserSession: @unchecked Sendable {
             }
         case "Page.frameStartedLoading":
             pauseRecordingCapture()
+            if let frameID = parameters["frameId"] as? String {
+                navigationLock.lock()
+                if mainFrameID == nil { mainFrameID = frameID }
+                if mainFrameID == frameID { navigationInProgress = true }
+                navigationLock.unlock()
+            }
+        case "Page.frameStoppedLoading":
+            if let frameID = parameters["frameId"] as? String {
+                navigationLock.lock()
+                if mainFrameID == frameID { navigationInProgress = false }
+                navigationLock.unlock()
+            }
+        case "Page.loadEventFired":
+            navigationLock.lock()
+            navigationInProgress = false
+            navigationLock.unlock()
         case "Page.frameRequestedNavigation":
             pauseRecordingCapture()
             if let frameID = parameters["frameId"] as? String,

apps/headless/LinuxHost/main.swift

@@ -77,6 +77,10 @@ final class ChromiumBrowserEngineSession: BrowserEngineSession {
 
     var hostIsolated: Bool { browserSession.isIsolated }
 
+    func hostSessionMetadata() throws -> BrowserSessionPageMetadata {
+        try browserSession.sessionMetadata()
+    }
+
     func hostVisit(_ url: URL) throws -> JSONValue { try browserSession.visit(url) }
     func hostInspect(parameters: [String: JSONValue]) throws -> JSONValue {
         try browserSession.inspect(parameters: parameters)
@@ -136,11 +140,11 @@ final class ChromiumBrowserEngineSession: BrowserEngineSession {
     }
     func hostQAReport() throws -> JSONValue { try browserSession.qaReport() }
     func hostQAClear() throws -> JSONValue { browserSession.diagnostics.clear() }
-    func hostConsole(level: String, limit: Int) throws -> JSONValue {
-        browserSession.console(level: level, limit: limit)
+    func hostConsole(level: String, limit: Int, cursor: String?) throws -> JSONValue {
+        try browserSession.console(level: level, limit: limit, cursor: cursor)
     }
-    func hostNetwork(failedOnly: Bool, status: Int?, limit: Int) throws -> JSONValue {
-        browserSession.network(failedOnly: failedOnly, status: status, limit: limit)
+    func hostNetwork(failedOnly: Bool, status: Int?, limit: Int, cursor: String?) throws -> JSONValue {
+        try browserSession.network(failedOnly: failedOnly, status: status, limit: limit, cursor: cursor)
     }
     func hostNetworkDetail(requestID: String) throws -> JSONValue {
         browserSession.networkDetail(requestID: requestID)

apps/headless/Sources/HeadlessCLI/main.swift

@@ -30,6 +30,81 @@ private struct HostLauncher {
 
     let client = LocalSocketClient()
 
+    private final class LogCapture {
+        let store: HostLogStore
+        let process: Process
+        let writer: FileHandle
+
+        init(environment: [String: String]) throws {
+            store = try HostLogStore(environment: environment)
+            try store.prepare()
+            let pipe = Pipe()
+            process = Process()
+            process.executableURL = try Self.runningCLIURL()
+            process.arguments = ["__host-log-writer"]
+            process.environment = [
+                "HEADLESS_INTERNAL_LOG_WRITER": "1",
+                "HEADLESS_INTERNAL_LOG_PATH": store.url.path,
+                "HEADLESS_INTERNAL_LOG_MAX_BYTES": String(store.maximumBytes),
+            ]
+            process.standardInput = pipe.fileHandleForReading
+            process.standardOutput = FileHandle.nullDevice
+            process.standardError = FileHandle.nullDevice
+            writer = pipe.fileHandleForWriting
+            do {
+                try process.run()
+                try pipe.fileHandleForReading.close()
+            } catch {
+                try? pipe.fileHandleForReading.close()
+                try? pipe.fileHandleForWriting.close()
+                throw error
+            }
+        }
+
+        func closeParentWriter() {
+            try? writer.close()
+        }
+
+        func waitForDrain() {
+            closeParentWriter()
+            let deadline = Date().addingTimeInterval(1)
+            while process.isRunning, Date() < deadline {
+                Thread.sleep(forTimeInterval: 0.01)
+            }
+            if process.isRunning {
+                process.terminate()
+                let terminationDeadline = Date().addingTimeInterval(1)
+                while process.isRunning, Date() < terminationDeadline {
+                    Thread.sleep(forTimeInterval: 0.01)
+                }
+            }
+            if process.isRunning { _ = kill(process.processIdentifier, SIGKILL) }
+            process.waitUntilExit()
+        }
+
+        var diagnostic: HostLaunchDiagnostic {
+            HostLaunchDiagnostic(path: store.url.path, tail: store.diagnosticTail())
+        }
+
+        private static func runningCLIURL() throws -> URL {
+            #if os(Linux)
+            let candidate = URL(fileURLWithPath: "/proc/self/exe").resolvingSymlinksInPath()
+            #else
+            var requiredSize: UInt32 = 0
+            _ = _NSGetExecutablePath(nil, &requiredSize)
+            var buffer = [CChar](repeating: 0, count: Int(requiredSize))
+            guard _NSGetExecutablePath(&buffer, &requiredSize) == 0 else {
+                throw HostLogError.operationFailed("writer executable resolution")
+            }
+            let candidate = URL(fileURLWithPath: String(cString: buffer)).standardizedFileURL
+            #endif
+            guard FileManager.default.isExecutableFile(atPath: candidate.path) else {
+                throw HostLogError.operationFailed("writer executable resolution")
+            }
+            return candidate
+        }
+    }
+
     func ping() -> CommandResponse? {
         try? client.send(CommandRequest(command: .ping), timeout: 0.5)
     }
@@ -75,22 +150,17 @@ private struct HostLauncher {
             environment.removeValue(forKey: headlessNavigationAllowlistEnvironmentKey)
         }
         process.environment = environment
+        let logCapture = try LogCapture(environment: environment)
         let ownerPipe = supervised ? Pipe() : nil
         process.standardInput = ownerPipe?.fileHandleForReading ?? FileHandle.nullDevice
-        if let hostLog = environment["HEADLESS_HOST_LOG"], hostLog.hasPrefix("/") {
-            let logURL = URL(fileURLWithPath: hostLog)
-            FileManager.default.createFile(atPath: logURL.path, contents: nil)
-            let handle = try FileHandle(forWritingTo: logURL)
-            process.standardOutput = handle
-            process.standardError = handle
-        } else {
-            process.standardOutput = FileHandle.nullDevice
-            process.standardError = FileHandle.nullDevice
-        }
+        process.standardOutput = logCapture.writer
+        process.standardError = logCapture.writer
         do {
             try process.run()
+            logCapture.closeParentWriter()
             try ownerPipe?.fileHandleForReading.close()
         } catch {
+            logCapture.waitForDrain()
             try? ownerPipe?.fileHandleForReading.close()
             try? ownerPipe?.fileHandleForWriting.close()
             throw error
@@ -117,13 +187,15 @@ private struct HostLauncher {
             if !process.isRunning {
                 try? ownerPipe?.fileHandleForWriting.close()
                 process.waitUntilExit()
-                throw HostLaunchError.exited(process.terminationStatus)
+                logCapture.waitForDrain()
+                throw HostLaunchError.exited(process.terminationStatus, logCapture.diagnostic)
             }
             Thread.sleep(forTimeInterval: 0.05)
         } while Date() < deadline
         try? ownerPipe?.fileHandleForWriting.close()
         terminateAndReap(process)
-        throw HostLaunchError.timedOut
+        logCapture.waitForDrain()
+        throw HostLaunchError.timedOut(logCapture.diagnostic)
     }
 
     func waitForSupervisedHost(_ launch: Launch) -> Int32 {
@@ -228,12 +300,22 @@ private struct HostLauncher {
     }
 }
 
+private struct HostLaunchDiagnostic {
+    let path: String
+    let tail: String?
+
+    var description: String {
+        guard let tail, !tail.isEmpty else { return "Host log: \(path)" }
+        return "Host log: \(path)\nRecent host output:\n\(tail)"
+    }
+}
+
 private enum HostLaunchError: Error, CustomStringConvertible {
     case notFound
     case alreadyRunning
     case ownershipMismatch
-    case timedOut
-    case exited(Int32)
+    case timedOut(HostLaunchDiagnostic)
+    case exited(Int32, HostLaunchDiagnostic)
     case allowlistMismatch(running: [String], requested: [String])
 
     var description: String {
@@ -243,8 +325,10 @@ private enum HostLaunchError: Error, CustomStringConvertible {
             return "A shared Headless host is already running. Stop it before starting a supervised host."
         case .ownershipMismatch:
             return "A different Headless host answered during supervised startup."
-        case .timedOut: return "Headless host did not become ready within 8 seconds."
-        case .exited(let status): return "Headless host exited during startup (status \(status))."
+        case .timedOut(let diagnostic):
+            return "Headless host did not become ready within 8 seconds.\n\(diagnostic.description)"
+        case .exited(let status, let diagnostic):
+            return "Headless host exited during startup (status \(status)).\n\(diagnostic.description)"
         case .allowlistMismatch(let running, let requested):
             let runningText = running.isEmpty ? "unrestricted" : running.joined(separator: ", ")
             return "The running host navigation allowlist (\(runningText)) does not match (\(requested.joined(separator: ", "))). Run `headless stop` first."
@@ -321,7 +405,30 @@ private enum CredentialBrokerLaunchError: Error, CustomStringConvertible {
     }
 }
 
+private func runInternalHostLogWriterIfRequested() -> Bool {
+    guard Array(CommandLine.arguments.dropFirst()) == ["__host-log-writer"] else { return false }
+    let environment = ProcessInfo.processInfo.environment
+    guard environment["HEADLESS_INTERNAL_LOG_WRITER"] == "1",
+          let path = environment["HEADLESS_INTERNAL_LOG_PATH"], path.hasPrefix("/"),
+          let rawMaximum = environment["HEADLESS_INTERNAL_LOG_MAX_BYTES"],
+          let maximum = Int(rawMaximum) else {
+        fputs("headless: internal log writer authorization failed\n", stderr)
+        exit(64)
+    }
+    do {
+        let store = try HostLogStore(
+            environment: ["HEADLESS_HOST_LOG": path], maximumBytes: maximum
+        )
+        try store.consume(.standardInput)
+    } catch {
+        fputs("headless: internal log writer failed\n", stderr)
+        exit(74)
+    }
+    return true
+}
+
 do {
+    if runInternalHostLogWriterIfRequested() { exit(0) }
     let invocation = try CLIParser().parse(Array(CommandLine.arguments.dropFirst()))
     if let local = invocation.local {
         switch local {
@@ -342,6 +449,10 @@ do {
                 "supported": .bool(true), "transport": .string("native-webkit"),
             ]))
             #endif
+        case .doctor:
+            let report = try HeadlessDoctor().run()
+            printJSON(report.document)
+            if report.hasFailures { exit(69) }
         case .start(let presentation, let allowlist, let supervised):
             let launch = try HostLauncher().start(
                 presentation: presentation, allowlist: allowlist, supervised: supervised
@@ -423,6 +534,13 @@ do {
     )
     try? printResponse(response)
     exit(69)
+} catch let error as HostLogError {
+    let response = CommandResponse.failure(
+        id: "unknown", code: "HOST_LOG_UNAVAILABLE", message: error.description,
+        suggestion: "Check the private runtime directory or HEADLESS_HOST_LOG path."
+    )
+    try? printResponse(response)
+    exit(69)
 } catch let error as SettingsError {
     let code: String
     switch error {