Skip to content
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
107 commits
Select commit Hold shift + click to select a range
e8646fd
feat: RemoteDiscovery module with Hyperswarm
Mar 24, 2026
2f14b72
fix: Connect to correct pear in remote discovery test
Mar 25, 2026
4255361
fix: Update bogon for hyperdht filtering
Mar 25, 2026
ad6c0be
test: Fix peer detection in remote discovery
Mar 25, 2026
f4a8c84
feat: Connect RemoteDiscovery to member API `inviteOverInternet` func…
Mar 25, 2026
8e2e3ac
feat: Simplify MemberAPI constructor parameters
Mar 26, 2026
039689f
test: Check member API invite over internet properties
Mar 26, 2026
b88f0fd
feat: RPC for redeeming invite over internet
Mar 26, 2026
f1f0767
feat: e2e invite over the internet
Mar 31, 2026
4675bdd
feat: Use ephemeral keys for hyperswarm, give real key on handshake
Apr 1, 2026
394facd
fix: MemberAPI.pendingInternetInvites should return URLs not invite IDs
Apr 1, 2026
01215f2
fix: Adjust remote discovery tests to use swarm key for connecting
Apr 2, 2026
ad82754
fix: Set own device ID for server tests since it is needed for projec…
Apr 2, 2026
54259c4
test: Check that remote connections can send data after connecting
Apr 2, 2026
50eaffb
fix: Use swarm public key to join project over internet
Apr 2, 2026
98b3350
chore: Clean handle redeem in member api
Apr 2, 2026
20203b0
fix: Pause swarm stream after doing initial handshake to avoid packet…
Apr 8, 2026
2b227d6
feat: Do not trust RPC until manual approval for swarm peers
Apr 8, 2026
968a737
chore: Add error classes for invite over internet code
Apr 9, 2026
d112c76
test: Check for error paths in RemoteDiscovery
Apr 13, 2026
f0026ac
fix: Detect existing connections
Apr 13, 2026
186d55b
test: Check handling of errors during invite over internet process
Apr 14, 2026
91b6614
chore: Fix formatting in manager constructor
Apr 14, 2026
225cb40
feat: Add new project table for pending internet invites
Apr 15, 2026
adccdcf
feat: Pending invites API with test
Apr 15, 2026
925e48a
fix: All ready resource methods can be optiona
Apr 15, 2026
dcc7c05
chore: Tidy remote discovery JSdoc types
Apr 15, 2026
35c4296
feat: Add persistent pending invites to member API
Apr 15, 2026
b789b56
fix: Type cast members events in tests due to ReadyResource change
Apr 15, 2026
e78ca86
test: Check for persistence in member API
Apr 15, 2026
e0264b1
feat: Length prefix remote discovery handshake buffer
Apr 20, 2026
6bc83d1
chore: merge with main
Apr 22, 2026
e9b7426
chore: Fix ready-resource removal issues
Apr 22, 2026
5cde04b
feat: Determenistic swarm key generation
Apr 22, 2026
922761a
fix: Use array.sort instead of array.toSorted for node18 support
Apr 22, 2026
f79759d
fix: log error when error is unknown type
Apr 23, 2026
138ba97
fix: Delete null values from loaded pending invite before passing to …
Apr 23, 2026
6c0122c
test: invite over the internet after invitor reloads app
Apr 23, 2026
3434e8c
feat: Store pending invites at the mapeo manager level
May 19, 2026
2d21931
Merge branch 'main' into feat/invite-over-internet
RangerMauve May 20, 2026
b0595b9
chore: move pending-invites-api into invite
May 20, 2026
c468f29
chore: denullify pending invite values
May 20, 2026
67b68a4
feat: Trust-gate more RPC methods
May 20, 2026
d3565ac
fix: Un-trust-gate redeem invite over internet
May 20, 2026
c37db5f
feat: Clear expired pending invites
May 21, 2026
7b6fe8b
feat: pending invites clear expired on init via ready-resource
May 21, 2026
3383b95
feat: Use timers to clear expired pending invites
May 21, 2026
b6ec012
chore: Remove in memory pending invites map
May 25, 2026
a93aa85
chore: Use swarmPublicKEy instead of deviceId in invite links
May 25, 2026
05d1268
chore: Fix timeout for handshake read chunk to be 10s
May 25, 2026
6a9bf28
fix: Clear member api listeners on project close
May 25, 2026
f319964
feat: Allow invite to be reused until it naturally expires
May 26, 2026
88dccaa
chore: Rename PendingInvites to InviteLinks
May 26, 2026
2b2defc
chore: refactor invite link params
May 26, 2026
987e672
chore: move pending invites api to invite links api
May 26, 2026
36b910a
feat: add expiresAt to invite links, expose in listInviteLinks
May 26, 2026
72e99a5
fix: Filter incoming invites by invite over internet invitor device ID
May 27, 2026
9553796
feat: Use HyperDHT testnet for RemoteDiscovery tests
May 27, 2026
f10b666
feat: Use testnet for invite over internet tests
May 27, 2026
38e9de2
fix: Track id from noise streams correctly.
May 27, 2026
19f9f8b
fix: Pull peer id from noise stream instead of peer remotePublicKey, …
May 27, 2026
8b413a8
feat: Encode more data in invite links, z32 for buffers instead of hex
Jun 1, 2026
e873ecd
feat: use crockford base32 for invite links
Jun 2, 2026
a1a9156
feat: cancel and deny invite over internet
Jun 4, 2026
2ecbae1
feat: Wire AbortSignal to RemoteDiscovery peer connection
Jun 4, 2026
eb7940b
fix: RemoteDiscovery Ensure connection is destroyed on disconnect
Jun 8, 2026
c36e919
feat: new leaveOnFail flag in Invite protobuf
Jun 8, 2026
2addca5
feat: new leaveOnFail flag in Invite protobuf
Jun 8, 2026
7a41460
feat: Leave project on fail, more robust initial sync timeout
Jun 8, 2026
0c829a9
test: InviteOverInternet, fail to sync and try again
Jun 8, 2026
6a06098
chore: use a.comapeo.app for invite over internet URL
Jun 8, 2026
3f919fa
feat: add seedTime to invite links, use oldest seed time or current t…
Jun 8, 2026
548d285
chore: rename pending-invites test to invite-links-api
Jun 8, 2026
f709900
docs: Guide for invite over internet
Jun 8, 2026
163d58c
fix: Wait for writes to finish before disconnecting from peer
Jun 10, 2026
8c3a752
chore: remove stray console logs
Jun 10, 2026
5020d7b
fix: Wait for sending ack send before emitting RPC event
Jun 10, 2026
668a6d2
feat: Emit invite-link-join-request without needing a project loaded
Jun 10, 2026
7ed2d2c
docs: Update invite over internet docs
Jun 10, 2026
2228e5d
feat: Distinguish reason between invitor deny and auto deny
Jun 10, 2026
3e5692d
test: Check that invite links seedTime works as expected
Jun 10, 2026
6f230a5
feat: invite link connected and accepted events on manager.
Jun 10, 2026
3b43a27
feat: allow list rpc methods on untrusted local peers
RangerMauve Jun 15, 2026
898cbc7
feat: gate sent rpc on trust, send device info on trusted peer
RangerMauve Jun 17, 2026
2c8bd3e
test: check for rejected RPC on untrusted peers
RangerMauve Jun 17, 2026
9c8177e
Merge branch 'main' into feat/invite-over-internet
RangerMauve Jun 17, 2026
570743b
fix: account for race conditions in finding existing peers in remote …
RangerMauve Jun 24, 2026
fd60941
fix: ensure remote connection ends on handshake error
RangerMauve Jun 24, 2026
08eeca3
feat: Disconnect peers if they don't get trusted within 10s
RangerMauve Jun 30, 2026
6514862
chore: remove only from untrusted timeout test
RangerMauve Jun 30, 2026
d10bb7b
feat: tighten restrictions on Ack rpc from untrusted peers
RangerMauve Jun 30, 2026
6f344ca
feat: Use StartStopStateMachine for RemoteDiscovery
RangerMauve Jun 30, 2026
eafa908
chore: Delete old keyBy util
RangerMauve Jun 30, 2026
4b699ce
feat: use 16bit length prefix and check for size being too large
RangerMauve Jun 30, 2026
15b0c5f
wip: InviteLinkJoiner API for refactoring joinProjectFromURL
RangerMauve Jul 2, 2026
d71cea0
test: fix tests for invite joiner
RangerMauve Jul 14, 2026
3bccac6
feat: replace old invite link API with new one
RangerMauve Jul 15, 2026
bffd48d
chore: fixes and cleanup for invite links api
RangerMauve Jul 15, 2026
8ba1699
fix: remote discovery should detect keypair rotation on swarm resume,…
RangerMauve Jul 15, 2026
104c72e
fix: leave peer on remote discovery connect peer timeout
RangerMauve Jul 15, 2026
0cd379a
chore: More robust allow list for sending RPC
RangerMauve Jul 15, 2026
4eafa5c
fix: Handle peer redeeming multiple invites over internet and disconn…
RangerMauve Jul 15, 2026
e4fa9c2
fix: simplify and stabilize member api
RangerMauve Jul 21, 2026
bc4747a
Merge branch 'main' into feat/invite-over-internet
RangerMauve Jul 21, 2026
b395f4d
feat: Use AuthedNoiseStream for both local and remote discovery
RangerMauve Jul 21, 2026
f27a6f4
feat: Allow setShouldListenOverInternet to be called more than once i…
RangerMauve Jul 21, 2026
60f76c0
fix: handle connections emitting finish instead of close, avoid infin…
RangerMauve Jul 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
475 changes: 452 additions & 23 deletions package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@
"bcp-47-normalize": "^2.3.0",
"better-sqlite3": "^11.10.0",
"big-sparse-array": "^1.0.3",
"bogon": "^1.1.0",
"bogon": "^1.2.0",
"comapeocat": "^1.0.0",
"compact-encoding": "^2.12.0",
"corestore": "6.8.4",
Expand All @@ -209,6 +209,7 @@
"hypercore": "10.19.0",
"hypercore-crypto": "3.4.2",
"hyperdrive": "11.5.3",
"hyperswarm": "^4.17.0",
"iso-3166": "^4.3.0",
"json-stable-stringify": "^1.1.1",
"magic-bytes.js": "^1.10.0",
Expand Down
8 changes: 8 additions & 0 deletions proto/rpc.proto
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ message ProjectJoinDetails {
EncryptionKeys encryptionKeys = 3;
}

message RedeemInviteOverInternet {
bytes inviteId = 1;
}

message DeviceInfo {
enum DeviceType {
device_type_unspecified = 0;
Expand Down Expand Up @@ -68,6 +72,10 @@ message ProjectJoinDetailsAck {
bytes inviteId = 1;
}

message RedeemInviteOverInternetAck {
bytes inviteId = 1;
}

message MapShareExtension {
// URLs to map share
repeated string mapShareUrls = 1;
Expand Down
113 changes: 113 additions & 0 deletions src/discovery/remote-discovery.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import { TypedEmitter } from 'tiny-typed-emitter'
import { Logger } from '../logger.js'
import Hyperswarm from 'hyperswarm'
import { pEvent } from 'p-event'

/** @import {OpenedNoiseStream} from '../lib/noise-secret-stream-helpers.js' */

/**
* @typedef {Object} DiscoveryEvents
* @property {(connection: OpenedNoiseStream) => void} connection
*/

/**
* @extends {TypedEmitter<DiscoveryEvents>}
*/
export class RemoteDiscovery extends TypedEmitter {
#l
/** @type {Hyperswarm?} */
#swarm = null
/** @type {Promise<Hyperswarm>?} */
#loading = null
#identityKeypair

/**
* @param {Object} opts
* @param {import('./local-discovery.js').Keypair} opts.identityKeypair
* @param {Logger} [opts.logger]
*/
constructor({ identityKeypair, logger }) {
super()
this.#l = Logger.create('RemoteDiscovery', logger)
this.#identityKeypair = identityKeypair
}

async #initSwarm() {
this.#l.log('Initializing swarm')

const swarm = new Hyperswarm({
keyPair: this.#identityKeypair,
maxPeers: 4,
Comment thread
RangerMauve marked this conversation as resolved.
Outdated
})
swarm.on('connection', this.#handleHyperswarmConnection.bind(this))
this.#l.log('Starting listen')
await swarm.listen()
this.#l.log('Listening')
return swarm
}

/**
* @returns {Promise<Hyperswarm>}
*/
async #ensureSwarm() {
await this.#loading
Comment thread
RangerMauve marked this conversation as resolved.
Outdated
if (!this.#swarm) {
this.#loading = this.#initSwarm()
this.#swarm = await this.#loading
}
return this.#swarm
}

/**
* Start listening for incoming connections
*/
async start() {
// TODO: Use start stop state machine
Comment thread
RangerMauve marked this conversation as resolved.
Outdated
const swarm = await this.#ensureSwarm()
await swarm.resume()
}
/**
* Close all connections and stop listening
*/
async stop() {
await this.#loading
await this.#swarm?.suspend()
}

async close() {
await this.#loading
await this.#swarm?.destroy()
this.#l.log('Closed swarm')
}

/**
* Connect to another peer by their NOISE public key
* @param {string} publicKey
* @param {object} [opts]
* @param {number} [opts.timeout]
*/
async connectPeer(publicKey, { timeout = 60_000 } = {}) {
const swarm = await this.#ensureSwarm()
const noisePublicKey = Buffer.from(publicKey, 'hex')

const onConnected = pEvent(this, 'connection', {
Comment thread
RangerMauve marked this conversation as resolved.
filter: (connection) => connection.remotePublicKey.equals(noisePublicKey),
timeout,
})
// Start trying to connect
swarm.joinPeer(noisePublicKey)
this.#l.log('Connecting to %S', publicKey)

const socket = await onConnected

return socket
}

/**
* @param {OpenedNoiseStream} socket
* @param {import('hyperswarm').PeerInfo} _peerInfo
*/
#handleHyperswarmConnection(socket, _peerInfo) {
this.emit('connection', socket)
}
}
18 changes: 18 additions & 0 deletions src/generated/rpc.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ export interface ProjectJoinDetails {
projectKey: Buffer;
encryptionKeys: EncryptionKeys | undefined;
}
export interface RedeemInviteOverInternet {
inviteId: Buffer;
}
export interface DeviceInfo {
name: string;
deviceType?: DeviceInfo_DeviceType | undefined;
Expand Down Expand Up @@ -69,6 +72,9 @@ export interface InviteResponseAck {
export interface ProjectJoinDetailsAck {
inviteId: Buffer;
}
export interface RedeemInviteOverInternetAck {
inviteId: Buffer;
}
export interface MapShareExtension {
/** URLs to map share */
mapShareUrls: string[];
Expand Down Expand Up @@ -117,6 +123,12 @@ export declare const ProjectJoinDetails: {
create<I extends Exact<DeepPartial<ProjectJoinDetails>, I>>(base?: I): ProjectJoinDetails;
fromPartial<I extends Exact<DeepPartial<ProjectJoinDetails>, I>>(object: I): ProjectJoinDetails;
};
export declare const RedeemInviteOverInternet: {
encode(message: RedeemInviteOverInternet, writer?: _m0.Writer): _m0.Writer;
decode(input: _m0.Reader | Uint8Array, length?: number): RedeemInviteOverInternet;
create<I extends Exact<DeepPartial<RedeemInviteOverInternet>, I>>(base?: I): RedeemInviteOverInternet;
fromPartial<I extends Exact<DeepPartial<RedeemInviteOverInternet>, I>>(object: I): RedeemInviteOverInternet;
};
export declare const DeviceInfo: {
encode(message: DeviceInfo, writer?: _m0.Writer): _m0.Writer;
decode(input: _m0.Reader | Uint8Array, length?: number): DeviceInfo;
Expand Down Expand Up @@ -147,6 +159,12 @@ export declare const ProjectJoinDetailsAck: {
create<I extends Exact<DeepPartial<ProjectJoinDetailsAck>, I>>(base?: I): ProjectJoinDetailsAck;
fromPartial<I extends Exact<DeepPartial<ProjectJoinDetailsAck>, I>>(object: I): ProjectJoinDetailsAck;
};
export declare const RedeemInviteOverInternetAck: {
encode(message: RedeemInviteOverInternetAck, writer?: _m0.Writer): _m0.Writer;
decode(input: _m0.Reader | Uint8Array, length?: number): RedeemInviteOverInternetAck;
create<I extends Exact<DeepPartial<RedeemInviteOverInternetAck>, I>>(base?: I): RedeemInviteOverInternetAck;
fromPartial<I extends Exact<DeepPartial<RedeemInviteOverInternetAck>, I>>(object: I): RedeemInviteOverInternetAck;
};
export declare const MapShareExtension: {
encode(message: MapShareExtension, writer?: _m0.Writer): _m0.Writer;
decode(input: _m0.Reader | Uint8Array, length?: number): MapShareExtension;
Expand Down
84 changes: 84 additions & 0 deletions src/generated/rpc.js
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,48 @@ export var ProjectJoinDetails = {
return message;
},
};
function createBaseRedeemInviteOverInternet() {
return { inviteId: Buffer.alloc(0) };
}
export var RedeemInviteOverInternet = {
encode: function (message, writer) {
if (writer === void 0) { writer = _m0.Writer.create(); }
if (message.inviteId.length !== 0) {
writer.uint32(10).bytes(message.inviteId);
}
return writer;
},
decode: function (input, length) {
var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
var end = length === undefined ? reader.len : reader.pos + length;
var message = createBaseRedeemInviteOverInternet();
while (reader.pos < end) {
var tag = reader.uint32();
switch (tag >>> 3) {
case 1:
if (tag !== 10) {
break;
}
message.inviteId = reader.bytes();
continue;
}
if ((tag & 7) === 4 || tag === 0) {
break;
}
reader.skipType(tag & 7);
}
return message;
},
create: function (base) {
return RedeemInviteOverInternet.fromPartial(base !== null && base !== void 0 ? base : {});
},
fromPartial: function (object) {
var _a;
var message = createBaseRedeemInviteOverInternet();
message.inviteId = (_a = object.inviteId) !== null && _a !== void 0 ? _a : Buffer.alloc(0);
return message;
},
};
function createBaseDeviceInfo() {
return { name: "", features: [] };
}
Expand Down Expand Up @@ -653,6 +695,48 @@ export var ProjectJoinDetailsAck = {
return message;
},
};
function createBaseRedeemInviteOverInternetAck() {
return { inviteId: Buffer.alloc(0) };
}
export var RedeemInviteOverInternetAck = {
encode: function (message, writer) {
if (writer === void 0) { writer = _m0.Writer.create(); }
if (message.inviteId.length !== 0) {
writer.uint32(10).bytes(message.inviteId);
}
return writer;
},
decode: function (input, length) {
var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
var end = length === undefined ? reader.len : reader.pos + length;
var message = createBaseRedeemInviteOverInternetAck();
while (reader.pos < end) {
var tag = reader.uint32();
switch (tag >>> 3) {
case 1:
if (tag !== 10) {
break;
}
message.inviteId = reader.bytes();
continue;
}
if ((tag & 7) === 4 || tag === 0) {
break;
}
reader.skipType(tag & 7);
}
return message;
},
create: function (base) {
return RedeemInviteOverInternetAck.fromPartial(base !== null && base !== void 0 ? base : {});
},
fromPartial: function (object) {
var _a;
var message = createBaseRedeemInviteOverInternetAck();
message.inviteId = (_a = object.inviteId) !== null && _a !== void 0 ? _a : Buffer.alloc(0);
return message;
},
};
function createBaseMapShareExtension() {
return {
mapShareUrls: [],
Expand Down
Loading