Use ldk-node payment pagination

Read payment pages directly from ldk-node so the server does not
store duplicate payment records. Keep the server database for
forwarded-payment history.

AI assistance: OpenAI Codex was used for this change.

Author: benthecarman

docs/configuration.md

@@ -209,11 +209,12 @@ Two resolution methods are supported via the `mode` field:
   <network>/                # e.g., bitcoin/, regtest/, signet/
     api_key                # API key
     ldk-server.log         # Log file
-    ldk_node_data.sqlite   # LDK Node state (channels, on-chain wallet)
-    ldk_server_data.sqlite # Payment and forwarding history
+    ldk_node_data.sqlite   # LDK Node state (channels, wallet, payments)
+    ldk_server_data.sqlite # Forwarded-payment history
 ```
 
 The mnemonic is the node's master secret, required to recover on-chain funds. On first start,
 ldk-server generates a fresh 24-word BIP39 mnemonic at `<storage_dir>/keys_mnemonic` if the file
-does not already exist. `ldk_node_data.sqlite` holds channel state, both are required to recover
-channel funds. See [Operations - Backups](operations.md#backups) for backup guidance.
+does not already exist. `ldk_node_data.sqlite` holds channel state and payment history. Both files
+are required to recover channel funds. See [Operations - Backups](operations.md#backups) for backup
+guidance.

docs/operations.md

@@ -53,8 +53,8 @@ the following config to `/etc/logrotate.d/ldk-server` (adjust the log path to ma
 | File                                   | Priority     | Description                                                                |
 | -------------------------------------- | ------------ | -------------------------------------------------------------------------- |
 | `<storage_dir>/keys_mnemonic`          | **Critical** | BIP39 mnemonic. Required to recover on-chain funds. Default for new installs. |
-| `<network_dir>/ldk_node_data.sqlite`   | **Critical** | Channel state and on-chain wallet data. Required to recover channel funds. |
-| `<network_dir>/ldk_server_data.sqlite` | Nice-to-have | Payment and forwarding history                                             |
+| `<network_dir>/ldk_node_data.sqlite`   | **Critical** | Channel state, on-chain wallet data, and payment history. Required to recover channel funds. |
+| `<network_dir>/ldk_server_data.sqlite` | Nice-to-have | Forwarded-payment history                                                  |
 
 ### What is Reconstructable
 

ldk-server/src/api/list_payments.rs

@@ -9,58 +9,25 @@
 
 use std::sync::Arc;
 
-use bytes::Bytes;
+use ldk_node::payment::PageToken as NodePageToken;
 use ldk_server_grpc::api::{ListPaymentsRequest, ListPaymentsResponse};
-use ldk_server_grpc::types::{PageToken, Payment};
-use prost::Message;
+use ldk_server_grpc::types::PageToken;
 
 use crate::api::error::LdkServerError;
-use crate::api::error::LdkServerErrorCode::InternalServerError;
-use crate::io::persist::{
-	PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE, PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE,
-};
 use crate::service::Context;
+use crate::util::proto_adapter::payment_to_proto;
 
 pub(crate) async fn handle_list_payments_request(
 	context: Arc<Context>, request: ListPaymentsRequest,
 ) -> Result<ListPaymentsResponse, LdkServerError> {
-	let page_token = request.page_token.map(|p| (p.token, p.index));
-	let list_response = context
-		.paginated_kv_store
-		.list(
-			PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE,
-			PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE,
-			page_token,
-		)
-		.map_err(|e| {
-			LdkServerError::new(InternalServerError, format!("Failed to list payments: {}", e))
-		})?;
+	let page_token = request.page_token.map(|p| NodePageToken::new(p.token));
+	let page = context.node.list_payments(page_token)?;
 
-	let mut payments: Vec<Payment> = Vec::with_capacity(list_response.keys.len());
-	for key in list_response.keys {
-		let payment_bytes = context
-			.paginated_kv_store
-			.read(
-				PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE,
-				PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE,
-				&key,
-			)
-			.map_err(|e| {
-				LdkServerError::new(
-					InternalServerError,
-					format!("Failed to read payment data: {}", e),
-				)
-			})?;
-		let payment = Payment::decode(Bytes::from(payment_bytes)).map_err(|e| {
-			LdkServerError::new(InternalServerError, format!("Failed to decode payment: {}", e))
-		})?;
-		payments.push(payment);
-	}
 	let response = ListPaymentsResponse {
-		payments,
-		next_page_token: list_response
+		payments: page.payments.into_iter().map(payment_to_proto).collect(),
+		next_page_token: page
 			.next_page_token
-			.map(|(token, index)| PageToken { token, index }),
+			.map(|token| PageToken { token: token.to_string(), index: 0 }),
 	};
 	Ok(response)
 }

ldk-server/src/io/persist/mod.rs

@@ -13,7 +13,3 @@ pub(crate) mod sqlite_store;
 /// The forwarded payments will be persisted under this prefix.
 pub(crate) const FORWARDED_PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE: &str = "forwarded_payments";
 pub(crate) const FORWARDED_PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE: &str = "";
-
-/// The payments will be persisted under this prefix.
-pub(crate) const PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE: &str = "payments";
-pub(crate) const PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE: &str = "";

ldk-server/src/main.rs

@@ -46,8 +46,7 @@ use crate::io::persist::paginated_kv_store::PaginatedKVStore;
 use crate::io::persist::sqlite_store::SqliteStore;
 use crate::io::persist::{
 	FORWARDED_PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE,
-	FORWARDED_PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE, PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE,
-	PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE,
+	FORWARDED_PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE,
 };
 use crate::service::NodeService;
 use crate::util::config::{load_config, ArgsConfig, ChainSource};
@@ -504,7 +503,7 @@ fn main() {
 								.map(node_to_proto_custom_tlv)
 								.collect();
 
-							send_event_and_upsert_payment(
+							send_payment_event(
 								&payment_id,
 								move |payment_ref| {
 									event_envelope::Event::PaymentReceived(events::PaymentReceived {
@@ -515,7 +514,6 @@ fn main() {
 								},
 								&event_node,
 								&event_sender,
-								Arc::clone(&paginated_store),
 							);
 
 							if let Some(metrics) = &metrics {
@@ -527,38 +525,36 @@ fn main() {
 							let bolt12_invoice = bolt12_invoice.as_ref().and_then(|invoice| {
 								invoice.bolt12_invoice().map(|i| i.encode().to_lower_hex_string())
 							});
-							send_event_and_upsert_payment(&payment_id,
+							send_payment_event(&payment_id,
 								|payment_ref| event_envelope::Event::PaymentSuccessful(events::PaymentSuccessful {
 									payment_id: payment_id.to_string(),
 									payment: Some(payment_ref.clone()),
 									payment_preimage,
 									bolt12_invoice,
 								}),
 								&event_node,
-								&event_sender,
-								Arc::clone(&paginated_store));
+								&event_sender);
 
 							if let Some(metrics) = &metrics {
 								metrics.update_payments_count(true);
 								metrics.update_all_balances(&event_node);
 							}
 						},
 						Event::PaymentFailed {payment_id, ..} => {
-							send_event_and_upsert_payment(&payment_id,
+							send_payment_event(&payment_id,
 								|payment_ref| event_envelope::Event::PaymentFailed(events::PaymentFailed {
 									payment_id: payment_id.to_string(),
 									payment: Some(payment_ref.clone()),
 								}),
 								&event_node,
-								&event_sender,
-								Arc::clone(&paginated_store));
+								&event_sender);
 
 							if let Some(metrics) = &metrics {
 								metrics.update_payments_count(false);
 							}
 						},
 						Event::PaymentClaimable { payment_id, custom_records, claim_deadline, .. } => {
-							send_event_and_upsert_payment(
+							send_payment_event(
 								&payment_id,
 								|payment_ref| {
 									event_envelope::Event::PaymentClaimable(
@@ -572,7 +568,6 @@ fn main() {
 								},
 								&event_node,
 								&event_sender,
-								Arc::clone(&paginated_store),
 							);
 						},
 						Event::PaymentForwarded {
@@ -706,10 +701,9 @@ fn main() {
 	log::logger().flush();
 }
 
-fn send_event_and_upsert_payment(
+fn send_payment_event(
 	payment_id: &PaymentId, payment_to_event: impl FnOnce(&Payment) -> event_envelope::Event,
 	event_node: &Node, event_sender: &broadcast::Sender<EventEnvelope>,
-	paginated_store: Arc<dyn PaginatedKVStore>,
 ) {
 	match event_node.payment(payment_id) {
 		Ok(Some(payment_details)) => {
@@ -720,7 +714,9 @@ fn send_event_and_upsert_payment(
 				debug!("No event subscribers connected, skipping event: {e}");
 			}
 
-			upsert_payment_details(event_node, Arc::clone(&paginated_store), &payment);
+			if let Err(e) = event_node.event_handled() {
+				error!("Failed to mark event as handled: {e}");
+			}
 		},
 		Ok(None) => error!("Unable to find payment with payment ID: {payment_id}"),
 		Err(e) => error!("Failed to retrieve payment with payment ID {payment_id}: {e}"),
@@ -879,30 +875,6 @@ fn closure_reason_details(
 	}
 }
 
-fn upsert_payment_details(
-	event_node: &Node, paginated_store: Arc<dyn PaginatedKVStore>, payment: &Payment,
-) {
-	let time =
-		SystemTime::now().duration_since(UNIX_EPOCH).expect("Time must be > 1970").as_secs() as i64;
-
-	match paginated_store.write(
-		PAYMENTS_PERSISTENCE_PRIMARY_NAMESPACE,
-		PAYMENTS_PERSISTENCE_SECONDARY_NAMESPACE,
-		&payment.payment_id,
-		time,
-		&payment.encode_to_vec(),
-	) {
-		Ok(_) => {
-			if let Err(e) = event_node.event_handled() {
-				error!("Failed to mark event as handled: {e}");
-			}
-		},
-		Err(e) => {
-			error!("Failed to write payment to persistence: {e}");
-		},
-	}
-}
-
 /// Loads the API key from a file, or generates a new one if it doesn't exist.
 /// The API key file is stored with 0400 permissions (read-only for owner).
 fn load_or_generate_api_key(storage_dir: &Path) -> std::io::Result<String> {