feat(connections): open a SQLite database that lives on an SSH server, read-only

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Data Rewind settings in Settings > Data & Results, with an off switch and Clear Saved Changes.
 - Restore Previous Values in the toolbar's Table Actions group.
 - Rebindable Find shortcut in Settings > Keyboard, for giving `Cmd+F` to the filter bar instead.
+- Remote File pane for SQLite, opening a read-only copy of a database that lives on an SSH server. (#2474)
 
 ### Changed
 

TablePro/Core/Database/CancellationFlag.swift

@@ -0,0 +1,26 @@
+//
+//  CancellationFlag.swift
+//  TablePro
+//
+
+import Foundation
+import os
+
+/// A cancellation signal a blocking loop can read.
+///
+/// `Task.isCancelled` is only visible to the task itself, and a transfer runs its libssh2 calls on a
+/// serial queue where that task does not exist. Flipping a flag from `withTaskCancellationHandler`
+/// gives the loop something it can check between chunks, which is the same shape the drivers use for
+/// a connect that cannot be interrupted mid-call.
+///
+/// This bounds cancellation at one chunk, not at one call: a read already inside libssh2 still has
+/// to return before the flag is looked at.
+final class CancellationFlag: @unchecked Sendable {
+    private let state = OSAllocatedUnfairLock(initialState: false)
+
+    var isCancelled: Bool { state.withLock { $0 } }
+
+    func cancel() {
+        state.withLock { $0 = true }
+    }
+}

TablePro/Core/Database/ConnectionTunnelError.swift

@@ -7,6 +7,8 @@ import Foundation
 
 enum ConnectionTunnelError: Error, LocalizedError, Equatable {
     case mutualExclusivityViolation([ConnectionTunnelKind])
+    case remoteFileUnsupported(String)
+    case remoteFilePathMissing
 
     var errorDescription: String? {
         switch self {
@@ -16,6 +18,13 @@ enum ConnectionTunnelError: Error, LocalizedError, Equatable {
                 format: String(localized: "A connection can use only one connection method at a time. Enabled: %@."),
                 names
             )
+        case .remoteFileUnsupported(let typeName):
+            return String(
+                format: String(localized: "%@ connections open a server, not a database file, so there is nothing to fetch over SSH."),
+                typeName
+            )
+        case .remoteFilePathMissing:
+            return String(localized: "No remote database file was named for this connection.")
         }
     }
 }

TablePro/Core/Database/DatabaseFileIntegrity.swift

@@ -0,0 +1,137 @@
+//
+//  DatabaseFileIntegrity.swift
+//  TablePro
+//
+
+import Foundation
+import os
+import SQLite3
+
+/// Checks that a file the app just downloaded is the database it claims to be.
+///
+/// This exists because of what SQLite does when it is not: `sqlite3_open` on a missing or empty
+/// path creates a valid, empty database and reports success. A download that was cancelled,
+/// truncated by a dropped session, or aimed at the wrong path would therefore open cleanly, show
+/// zero tables, and be indistinguishable from a database that really is empty. Writing that back
+/// destroys the remote file, and every step up to it reports success.
+///
+/// A byte count is not enough on its own either: SFTP writes a file front to back, so a partial
+/// download carries an intact header and a plausible prefix.
+enum DatabaseFileIntegrity {
+    private static let logger = Logger(subsystem: "com.TablePro", category: "RemoteDatabaseFile")
+
+    /// The 16 bytes every SQLite database starts with, including the trailing NUL.
+    private static let sqliteMagic = Array("SQLite format 3\u{0}".utf8)
+
+    /// DuckDB writes this at offset 8 of its first page.
+    private static let duckdbMagic = Array("DUCK".utf8)
+
+    enum Verdict: Equatable {
+        case ok
+        case wrongSize(expected: UInt64, actual: UInt64)
+        case notADatabase
+        case corrupt(String)
+
+        var isOK: Bool { self == .ok }
+    }
+
+    /// Verifies a downloaded artifact against what the server said it was sending.
+    ///
+    /// `expectedBytes` is the size reported by the remote stat, and `expectedSHA256` the hash of
+    /// what actually arrived. Both are compared before the file is treated as a database at all,
+    /// because a size mismatch names a truncated transfer more precisely than any parse can.
+    static func verifyDownload(
+        at url: URL,
+        expectedBytes: UInt64,
+        runsIntegrityCheck: Bool
+    ) -> Verdict {
+        let actual = (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? UInt64) ?? nil
+        guard let actual else { return .notADatabase }
+        guard actual == expectedBytes else {
+            return .wrongSize(expected: expectedBytes, actual: actual)
+        }
+        guard actual > 0 else { return .notADatabase }
+        guard let kind = fileKind(at: url) else { return .notADatabase }
+
+        guard runsIntegrityCheck, kind == .sqlite else { return .ok }
+        return integrityCheck(at: url)
+    }
+
+    enum FileKind: Equatable {
+        case sqlite
+        case duckdb
+        case text
+    }
+
+    /// Reads enough of the first page to tell what wrote the file. A ledger is plain text and has
+    /// no magic, so anything that is not a known binary format is reported as text rather than
+    /// rejected.
+    static func fileKind(at url: URL) -> FileKind? {
+        guard let handle = try? FileHandle(forReadingFrom: url) else { return nil }
+        defer { try? handle.close() }
+        guard let head = try? handle.read(upToCount: 32), !head.isEmpty else { return nil }
+
+        let bytes = [UInt8](head)
+        if bytes.count >= sqliteMagic.count, Array(bytes.prefix(sqliteMagic.count)) == sqliteMagic {
+            return .sqlite
+        }
+        if bytes.count >= 12, Array(bytes[8..<12]) == duckdbMagic {
+            return .duckdb
+        }
+        return .text
+    }
+
+    /// Runs `PRAGMA integrity_check` and reports the first thing it complains about.
+    ///
+    /// Opened read-only and with an immutable URI so the check cannot itself create, modify, or
+    /// replay a journal against the file it is inspecting.
+    static func integrityCheck(at url: URL) -> Verdict {
+        var handle: OpaquePointer?
+        let uri = "file:\(url.path)?mode=ro&immutable=1"
+        guard sqlite3_open_v2(uri, &handle, SQLITE_OPEN_READONLY | SQLITE_OPEN_URI, nil) == SQLITE_OK,
+              let handle
+        else {
+            if let handle { sqlite3_close(handle) }
+            return .notADatabase
+        }
+        defer { sqlite3_close(handle) }
+
+        var statement: OpaquePointer?
+        guard sqlite3_prepare_v2(handle, "PRAGMA integrity_check(1)", -1, &statement, nil) == SQLITE_OK else {
+            return .corrupt(String(cString: sqlite3_errmsg(handle)))
+        }
+        defer { sqlite3_finalize(statement) }
+
+        guard sqlite3_step(statement) == SQLITE_ROW, let text = sqlite3_column_text(statement, 0) else {
+            return .corrupt(String(cString: sqlite3_errmsg(handle)))
+        }
+        let result = String(cString: text)
+        guard result == "ok" else {
+            Self.logger.error("integrity_check on the working copy said \(result, privacy: .public)")
+            return .corrupt(result)
+        }
+        return .ok
+    }
+
+    /// Folds a write-ahead log back into the main file so a single-file upload carries every
+    /// committed row.
+    ///
+    /// A driver that implements `closeAndFlush()` has already done this. Running it again costs one
+    /// open on a file nobody holds and closes the gap for a driver that has not, which is the
+    /// difference between uploading the user's edits and uploading the state before them.
+    @discardableResult
+    static func checkpointWriteAheadLog(at url: URL) -> Bool {
+        var handle: OpaquePointer?
+        guard sqlite3_open_v2(url.path, &handle, SQLITE_OPEN_READWRITE, nil) == SQLITE_OK, let handle else {
+            if let handle { sqlite3_close(handle) }
+            return false
+        }
+        defer { sqlite3_close(handle) }
+
+        var statement: OpaquePointer?
+        guard sqlite3_prepare_v2(handle, "PRAGMA wal_checkpoint(TRUNCATE)", -1, &statement, nil) == SQLITE_OK
+        else { return false }
+        defer { sqlite3_finalize(statement) }
+        return sqlite3_step(statement) == SQLITE_ROW
+    }
+}

TablePro/Core/Database/DatabaseManager+Queries.swift

@@ -39,6 +39,11 @@ extension DatabaseManager {
         sshPassword: String? = nil,
         passwordOverride: String? = nil
     ) async throws -> Bool {
+        // A remote file answers the only question this button asks without moving the file.
+        if connection.activeTunnelKind == .remoteFile {
+            return try await testRemoteDatabaseFile(connection, sshPassword: sshPassword)
+        }
+
         // Build effective connection (creates SSH tunnel if needed)
         let testConnection = try await buildEffectiveConnection(
             for: connection,

TablePro/Core/Database/DatabaseManager+RemoteFile.swift

@@ -0,0 +1,144 @@
+//
+//  DatabaseManager+RemoteFile.swift
+//  TablePro
+//
+
+import Foundation
+import os
+import TableProPluginKit
+
+extension DatabaseManager {
+    /// Rewrites a file-backed connection to open a local working copy of a file that lives on an
+    /// SSH server.
+    ///
+    /// The symmetry with the tunnel arm is the point: that one swaps `host` and `port` for the
+    /// forwarded port and hands the driver a connection with no idea a tunnel exists, and this one
+    /// swaps the path. A driver never learns its database came from somewhere else, which is what
+    /// keeps every file-backed plugin working without changes.
+    internal func buildRemoteFileEffectiveConnection(
+        for connection: DatabaseConnection,
+        sshPasswordOverride: String? = nil
+    ) async throws -> DatabaseConnection {
+        let sshConfig = connection.resolvedSSHConfig
+        guard let field = pluginManager.localFilePathField(for: connection.type) else {
+            throw ConnectionTunnelError.remoteFileUnsupported(connection.type.displayName)
+        }
+
+        let credentials = sshCredentials(for: connection, passwordOverride: sshPasswordOverride)
+        let identity = RemoteFileIdentity(
+            username: sshConfig.username,
+            host: sshConfig.host,
+            port: sshConfig.port ?? 22,
+            path: sshConfig.remoteFilePath
+        )
+
+        let file = try await RemoteFileTransportManager.shared.materialize(
+            connectionId: connection.id,
+            identity: identity,
+            config: sshConfig,
+            credentials: credentials,
+            layout: DatabaseFileLayout.forType(connection.type),
+            forceRefetch: false
+        )
+
+        var effective = connection.substitutingLocalFilePath(file.workingCopy.path, in: field)
+
+        // Read-only is enforced here rather than promised in the pane's copy. The driver opens a
+        // copy on this Mac, so an edit would succeed locally, change nothing on the server, and be
+        // discarded the next time the file is fetched. Routing it through the same `safeModeLevel`
+        // the rest of the app already honours means the grid, the editor and the AI tools all
+        // refuse the write for the same reason, instead of each having to learn about remote files.
+        effective.safeModeLevel = .readOnly
+        return effective
+    }
+
+    /// Answers Test Connection without fetching the database.
+    ///
+    /// The button asks three things: does the server accept these credentials, is the file there,
+    /// and can this account read it. All three are settled by opening the session and asking for the
+    /// file's attributes. Routing the test through the ordinary connect path instead would download
+    /// the whole database before the button could report anything, which on a large one is minutes
+    /// of transfer to answer a question that took one round trip.
+    internal func testRemoteDatabaseFile(
+        _ connection: DatabaseConnection,
+        sshPassword: String?
+    ) async throws -> Bool {
+        let sshConfig = connection.resolvedSSHConfig
+        guard !sshConfig.remoteFilePath.isEmpty else { throw ConnectionTunnelError.remoteFilePathMissing }
+
+        let session = try await LibSSH2SFTPSession.open(
+            config: sshConfig,
+            credentials: sshCredentials(for: connection, passwordOverride: sshPassword),
+            label: "test-\(connection.id.uuidString)"
+        )
+        defer { session.close() }
+
+        let path = try session.resolvedPath(sshConfig.remoteFilePath)
+        let stat = try session.stat(path)
+        guard !stat.isDirectory else { throw SFTPError.notAFile(path: path) }
+        return true
+    }
+
+    internal func materializedRemoteFile(for connectionId: UUID) async -> MaterializedRemoteFile? {
+        await RemoteFileTransportManager.shared.existingFile(for: connectionId)
+    }
+
+    internal func sshCredentials(
+        for connection: DatabaseConnection,
+        passwordOverride: String?
+    ) -> SSHTunnelCredentials {
+        let storedPassword: String?
+        let keyPassphrase: String?
+        let totpSecret: String?
+
+        switch connection.sshTunnelMode {
+        case .disabled:
+            storedPassword = nil
+            keyPassphrase = nil
+            totpSecret = nil
+        case .profile(let profileId, _):
+            storedPassword = SSHProfileStorage.shared.loadSSHPassword(for: profileId)
+            keyPassphrase = SSHProfileStorage.shared.loadKeyPassphrase(for: profileId)
+            totpSecret = SSHProfileStorage.shared.loadTOTPSecret(for: profileId)
+        case .inline:
+            storedPassword = connectionStorage.loadSSHPassword(for: connection.id)
+            keyPassphrase = connectionStorage.loadKeyPassphrase(for: connection.id)
+            totpSecret = connectionStorage.loadTOTPSecret(for: connection.id)
+        }
+
+        return SSHTunnelCredentials(
+            sshPassword: passwordOverride ?? storedPassword,
+            keyPassphrase: keyPassphrase,
+            totpSecret: totpSecret,
+            keyboardInteractivePromptProvider: nil
+        )
+    }
+}
+
+extension DatabaseConnection {
+    /// Returns a copy whose driver will open `path`, written into whichever field this driver reads.
+    ///
+    /// SQLite and Beancount take the built-in `database`; DuckDB and libSQL keep their path in a
+    /// plugin-declared additional field and leave `database` empty, which is why the field cannot
+    /// be assumed.
+    func substitutingLocalFilePath(_ path: String, in field: LocalFilePathField) -> DatabaseConnection {
+        var copy = self
+        switch field {
+        case .database:
+            copy.database = path
+        case .additionalField(let id):
+            copy.additionalFields[id] = path
+        }
+        return copy
+    }
+
+    /// The path the driver will open, read from wherever this driver keeps it.
+    func localFilePath(in field: LocalFilePathField) -> String {
+        switch field {
+        case .database:
+            return database
+        case .additionalField(let id):
+            return additionalFields[id] ?? ""
+        }
+    }
+}

TablePro/Core/Database/DatabaseManager+SSH.swift

@@ -33,6 +33,11 @@ extension DatabaseManager {
             return try await buildCloudSQLProxyEffectiveConnection(for: connection)
         case .socksProxy:
             return try await buildSOCKSProxyEffectiveConnection(for: connection)
+        case .remoteFile:
+            return try await buildRemoteFileEffectiveConnection(
+                for: connection,
+                sshPasswordOverride: sshPasswordOverride
+            )
         case .ssh, .none:
             break
         }

TablePro/Core/Database/DatabaseManager+SystemEvents.swift

@@ -52,6 +52,8 @@ extension DatabaseManager {
                 await handleCloudSQLProxyTunnelDied(connectionId: connectionId)
             case .socksProxy:
                 await handleSOCKSProxyTunnelDied(connectionId: connectionId)
+            case .remoteFile:
+                break
             }
         }
     }

TablePro/Core/Database/DatabaseManager+Tunnel.swift

@@ -99,6 +99,7 @@ extension DatabaseManager {
         case .cloudflare: return CloudflareTunnelManager.shared
         case .cloudSQLProxy: return CloudSQLProxyManager.shared
         case .socksProxy: return SOCKSProxyManager.shared
+        case .remoteFile: return RemoteFileTransportManager.shared
         case .none: return nil
         }
     }