Migrate payment history to node pagination

Read payment history directly from LDK Node and stop duplicating
payment records in the server database. Retain the server store for
forwarded-payment history.

Replace structured page tokens with opaque strings across both listing
APIs and adapt CLI pagination to pass them through. Existing clients
must update for the token schema change.

AI assistance: OpenAI Codex was used for this change.

Author: benthecarman

docs/api-guide.md

@@ -271,8 +271,11 @@ force-closure of the channel.
 
 `ListPayments` and `ListForwardedPayments` support cursor-based pagination:
 
-1. Make the first request with your desired `number_of_payments` page size.
+1. Make the first request without a `page_token`. The server controls the page size.
 2. If the response includes a `next_page_token`, pass it as `page_token` in the next request.
 3. When `next_page_token` is absent, you have reached the end of the results.
 
-Results are ordered by creation time (most recent first).
+The page token is one opaque string. Do not parse or modify it. Results are ordered by creation
+time (most recent first).
+
+The CLI `--number-of-payments` option combines multiple pages. It does not set the gRPC page size.

docs/configuration.md

@@ -209,11 +209,12 @@ Two resolution methods are supported via the `mode` field:
   <network>/                # e.g., bitcoin/, regtest/, signet/
     api_key                # API key
     ldk-server.log         # Log file
-    ldk_node_data.sqlite   # LDK Node state (channels, on-chain wallet)
-    ldk_server_data.sqlite # Payment and forwarding history
+    ldk_node_data.sqlite   # LDK Node state (channels, wallet, payments)
+    ldk_server_data.sqlite # Forwarded-payment history
 ```
 
 The mnemonic is the node's master secret, required to recover on-chain funds. On first start,
 ldk-server generates a fresh 24-word BIP39 mnemonic at `<storage_dir>/keys_mnemonic` if the file
-does not already exist. `ldk_node_data.sqlite` holds channel state, both are required to recover
-channel funds. See [Operations - Backups](operations.md#backups) for backup guidance.
+does not already exist. `ldk_node_data.sqlite` holds channel state and payment history. Both files
+are required to recover channel funds. See [Operations - Backups](operations.md#backups) for backup
+guidance.

docs/operations.md

@@ -53,8 +53,8 @@ the following config to `/etc/logrotate.d/ldk-server` (adjust the log path to ma
 | File                                   | Priority     | Description                                                                |
 | -------------------------------------- | ------------ | -------------------------------------------------------------------------- |
 | `<storage_dir>/keys_mnemonic`          | **Critical** | BIP39 mnemonic. Required to recover on-chain funds. Default for new installs. |
-| `<network_dir>/ldk_node_data.sqlite`   | **Critical** | Channel state and on-chain wallet data. Required to recover channel funds. |
-| `<network_dir>/ldk_server_data.sqlite` | Nice-to-have | Payment and forwarding history                                             |
+| `<network_dir>/ldk_node_data.sqlite`   | **Critical** | Channel state, on-chain wallet data, and payment history. Required to recover channel funds. |
+| `<network_dir>/ldk_server_data.sqlite` | Nice-to-have | Forwarded-payment history                                                  |
 
 ### What is Reconstructable
 

ldk-server-cli/src/main.rs

@@ -50,7 +50,7 @@ use ldk_server_client::ldk_server_grpc::api::{
 };
 use ldk_server_client::ldk_server_grpc::types::{
 	bolt11_invoice_description, Bolt11InvoiceDescription, ChannelConfig, CustomTlvRecord,
-	PageToken, PayerProofOptions, RouteParametersConfig,
+	PayerProofOptions, RouteParametersConfig,
 };
 use ldk_server_client::{
 	DEFAULT_EXPIRY_SECS, DEFAULT_MAX_CHANNEL_SATURATION_POWER_OF_HALF, DEFAULT_MAX_PATH_COUNT,
@@ -540,7 +540,7 @@ enum Commands {
 		)]
 		number_of_payments: Option<u64>,
 		#[arg(long)]
-		#[arg(help = "Page token to continue from a previous page (format: token:index)")]
+		#[arg(help = "Opaque page token returned by a previous request")]
 		page_token: Option<String>,
 	},
 	#[command(about = "Get details of a specific payment by its payment ID")]
@@ -556,7 +556,7 @@ enum Commands {
 			help = "Fetch at least this many forwarded payments by iterating through multiple pages. Returns combined results with the last page token. If not provided, returns only a single page."
 		)]
 		number_of_payments: Option<u64>,
-		#[arg(long, help = "Page token to continue from a previous page (format: token:index)")]
+		#[arg(long, help = "Opaque page token returned by a previous request")]
 		page_token: Option<String>,
 	},
 	#[command(about = "Update the forwarding fees and CLTV expiry delta for an existing channel")]
@@ -1174,9 +1174,6 @@ async fn main() {
 			);
 		},
 		Commands::ListPayments { number_of_payments, page_token } => {
-			let page_token = page_token
-				.map(|token_str| parse_page_token(&token_str).unwrap_or_else(|e| handle_error(e)));
-
 			handle_response_result::<_, CliListPaymentsResponse>(
 				fetch_paginated(
 					number_of_payments,
@@ -1193,9 +1190,6 @@ async fn main() {
 			);
 		},
 		Commands::ListForwardedPayments { number_of_payments, page_token } => {
-			let page_token = page_token
-				.map(|token_str| parse_page_token(&token_str).unwrap_or_else(|e| handle_error(e)));
-
 			handle_response_result::<_, CliListForwardedPaymentsResponse>(
 				fetch_paginated(
 					number_of_payments,
@@ -1334,9 +1328,8 @@ fn build_open_channel_config(
 }
 
 async fn fetch_paginated<T, R, Fut>(
-	target_count: Option<u64>, initial_page_token: Option<PageToken>,
-	fetch_page: impl Fn(Option<PageToken>) -> Fut,
-	extract: impl Fn(R) -> (Vec<T>, Option<PageToken>),
+	target_count: Option<u64>, initial_page_token: Option<String>,
+	fetch_page: impl Fn(Option<String>) -> Fut, extract: impl Fn(R) -> (Vec<T>, Option<String>),
 ) -> Result<CliPaginatedResponse<T>, LdkServerError>
 where
 	Fut: std::future::Future<Output = Result<R, LdkServerError>>,
@@ -1446,20 +1439,6 @@ fn parse_bolt11_invoice_description(
 	}
 }
 
-fn parse_page_token(token_str: &str) -> Result<PageToken, LdkServerError> {
-	let parts: Vec<&str> = token_str.split(':').collect();
-	if parts.len() != 2 {
-		return Err(LdkServerError::new(
-			InvalidRequestError,
-			"Page token must be in format 'token:index'".to_string(),
-		));
-	}
-	let index = parts[1].parse::<i64>().map_err(|_| {
-		LdkServerError::new(InvalidRequestError, "Invalid page token index".to_string())
-	})?;
-	Ok(PageToken { token: parts[0].to_string(), index })
-}
-
 fn parse_custom_tlv(s: &str) -> Result<(u64, Vec<u8>), String> {
 	let (type_str, hex_str) =
 		s.split_once(':').ok_or_else(|| format!("expected <type_num>:<hex_value>, got '{s}'"))?;
@@ -1494,6 +1473,31 @@ fn handle_error(e: LdkServerError) -> ! {
 mod tests {
 	use super::*;
 
+	#[tokio::test]
+	async fn fetch_paginated_collects_multiple_pages() {
+		let response = fetch_paginated(
+			Some(3),
+			None,
+			|page_token| async move {
+				match page_token {
+					None => {
+						Ok::<_, LdkServerError>((vec![1, 2], Some("store:v2:cursor:7".to_string())))
+					},
+					Some(token) => {
+						assert_eq!(token, "store:v2:cursor:7");
+						Ok((vec![3], None))
+					},
+				}
+			},
+			|response| response,
+		)
+		.await
+		.unwrap();
+
+		assert_eq!(response.list, vec![1, 2, 3]);
+		assert!(response.next_page_token.is_none());
+	}
+
 	#[test]
 	fn parse_custom_tlv_accepts_valid_record() {
 		let (type_num, value) = parse_custom_tlv("65537:deadbeef").unwrap();

ldk-server-cli/src/types.rs

@@ -17,11 +17,11 @@ use std::fmt;
 use std::str::FromStr;
 
 use hex_conservative::{DisplayHex, FromHex};
-use ldk_server_client::ldk_server_grpc::types::{ForwardedPayment, PageToken, Payment};
+use ldk_server_client::ldk_server_grpc::types::{ForwardedPayment, Payment};
 use serde::Serialize;
 
-/// CLI-specific wrapper for paginated responses that formats the page token
-/// as "token:idx" instead of a JSON object.
+/// CLI-specific wrapper for paginated responses that keeps the page token as
+/// one opaque string.
 #[derive(Debug, Clone, Serialize)]
 pub struct CliPaginatedResponse<T> {
 	/// List of items.
@@ -32,18 +32,14 @@ pub struct CliPaginatedResponse<T> {
 }
 
 impl<T> CliPaginatedResponse<T> {
-	pub fn new(list: Vec<T>, next_page_token: Option<PageToken>) -> Self {
-		Self { list, next_page_token: next_page_token.map(format_page_token) }
+	pub fn new(list: Vec<T>, next_page_token: Option<String>) -> Self {
+		Self { list, next_page_token }
 	}
 }
 
 pub type CliListPaymentsResponse = CliPaginatedResponse<Payment>;
 pub type CliListForwardedPaymentsResponse = CliPaginatedResponse<ForwardedPayment>;
 
-fn format_page_token(token: PageToken) -> String {
-	format!("{}:{}", token.token, token.index)
-}
-
 /// A denomination-aware amount that stores its value internally in millisatoshis.
 ///
 /// Accepts the following formats when parsed from a string:

ldk-server-grpc/src/api.rs

@@ -896,14 +896,14 @@ pub struct GetPaymentDetailsResponse {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ListPaymentsRequest {
-	/// `page_token` is a pagination token.
+	/// `page_token` is an opaque pagination token string.
 	///
 	/// To query for the first page, `page_token` must not be specified.
 	///
 	/// For subsequent pages, use the value that was returned as `next_page_token` in the previous
 	/// page's response.
-	#[prost(message, optional, tag = "1")]
-	pub page_token: ::core::option::Option<super::types::PageToken>,
+	#[prost(string, optional, tag = "1")]
+	pub page_token: ::core::option::Option<::prost::alloc::string::String>,
 }
 /// The response for the `ListPayments` RPC. On failure, a gRPC error status is returned.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
@@ -915,7 +915,8 @@ pub struct ListPaymentsResponse {
 	/// List of payments.
 	#[prost(message, repeated, tag = "1")]
 	pub payments: ::prost::alloc::vec::Vec<super::types::Payment>,
-	/// `next_page_token` is a pagination token, used to retrieve the next page of results.
+	/// `next_page_token` is an opaque pagination token string used to retrieve the next page of
+	/// results.
 	/// Use this value to query for next-page of paginated operation, by specifying
 	/// this value as the `page_token` in the next request.
 	///
@@ -928,8 +929,8 @@ pub struct ListPaymentsResponse {
 	///
 	/// **Caution**: Clients must not assume a specific number of records to be present in a page for
 	/// paginated response.
-	#[prost(message, optional, tag = "2")]
-	pub next_page_token: ::core::option::Option<super::types::PageToken>,
+	#[prost(string, optional, tag = "2")]
+	pub next_page_token: ::core::option::Option<::prost::alloc::string::String>,
 }
 /// Retrieves list of all forwarded payments.
 /// See more: <https://docs.rs/ldk-node/latest/ldk_node/enum.Event.html#variant.PaymentForwarded>
@@ -939,14 +940,14 @@ pub struct ListPaymentsResponse {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ListForwardedPaymentsRequest {
-	/// `page_token` is a pagination token.
+	/// `page_token` is an opaque pagination token string.
 	///
 	/// To query for the first page, `page_token` must not be specified.
 	///
 	/// For subsequent pages, use the value that was returned as `next_page_token` in the previous
 	/// page's response.
-	#[prost(message, optional, tag = "1")]
-	pub page_token: ::core::option::Option<super::types::PageToken>,
+	#[prost(string, optional, tag = "1")]
+	pub page_token: ::core::option::Option<::prost::alloc::string::String>,
 }
 /// The response for the `ListForwardedPayments` RPC. On failure, a gRPC error status is returned.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
@@ -958,7 +959,8 @@ pub struct ListForwardedPaymentsResponse {
 	/// List of forwarded payments.
 	#[prost(message, repeated, tag = "1")]
 	pub forwarded_payments: ::prost::alloc::vec::Vec<super::types::ForwardedPayment>,
-	/// `next_page_token` is a pagination token, used to retrieve the next page of results.
+	/// `next_page_token` is an opaque pagination token string used to retrieve the next page of
+	/// results.
 	/// Use this value to query for next-page of paginated operation, by specifying
 	/// this value as the `page_token` in the next request.
 	///
@@ -971,8 +973,8 @@ pub struct ListForwardedPaymentsResponse {
 	///
 	/// **Caution**: Clients must not assume a specific number of records to be present in a page for
 	/// paginated response.
-	#[prost(message, optional, tag = "2")]
-	pub next_page_token: ::core::option::Option<super::types::PageToken>,
+	#[prost(string, optional, tag = "2")]
+	pub next_page_token: ::core::option::Option<::prost::alloc::string::String>,
 }
 /// Sign a message with the node's secret key.
 /// See more: <https://docs.rs/ldk-node/latest/ldk_node/struct.Node.html#method.sign_message>

ldk-server-grpc/src/proto/api.proto

@@ -650,21 +650,22 @@ message GetPaymentDetailsResponse {
 // Retrieves list of all payments.
 // See more: https://docs.rs/ldk-node/latest/ldk_node/struct.Node.html#method.list_payments
 message ListPaymentsRequest {
-  // `page_token` is a pagination token.
+  // `page_token` is an opaque pagination token string.
   //
   // To query for the first page, `page_token` must not be specified.
   //
   // For subsequent pages, use the value that was returned as `next_page_token` in the previous
   // page's response.
-  optional types.PageToken page_token = 1;
+  optional string page_token = 1;
 }
 
 // The response for the `ListPayments` RPC. On failure, a gRPC error status is returned.
 message ListPaymentsResponse {
   // List of payments.
   repeated types.Payment payments = 1;
 
-  // `next_page_token` is a pagination token, used to retrieve the next page of results.
+  // `next_page_token` is an opaque pagination token string used to retrieve the next page of
+  // results.
   // Use this value to query for next-page of paginated operation, by specifying
   // this value as the `page_token` in the next request.
   //
@@ -677,27 +678,28 @@ message ListPaymentsResponse {
   //
   // **Caution**: Clients must not assume a specific number of records to be present in a page for
   // paginated response.
-  optional types.PageToken next_page_token = 2;
+  optional string next_page_token = 2;
 }
 
 // Retrieves list of all forwarded payments.
 // See more: https://docs.rs/ldk-node/latest/ldk_node/enum.Event.html#variant.PaymentForwarded
 message ListForwardedPaymentsRequest {
-  // `page_token` is a pagination token.
+  // `page_token` is an opaque pagination token string.
   //
   // To query for the first page, `page_token` must not be specified.
   //
   // For subsequent pages, use the value that was returned as `next_page_token` in the previous
   // page's response.
-  optional types.PageToken page_token = 1;
+  optional string page_token = 1;
 }
 
 // The response for the `ListForwardedPayments` RPC. On failure, a gRPC error status is returned.
 message ListForwardedPaymentsResponse {
   // List of forwarded payments.
   repeated types.ForwardedPayment forwarded_payments = 1;
 
-  // `next_page_token` is a pagination token, used to retrieve the next page of results.
+  // `next_page_token` is an opaque pagination token string used to retrieve the next page of
+  // results.
   // Use this value to query for next-page of paginated operation, by specifying
   // this value as the `page_token` in the next request.
   //
@@ -710,7 +712,7 @@ message ListForwardedPaymentsResponse {
   //
   // **Caution**: Clients must not assume a specific number of records to be present in a page for
   // paginated response.
-  optional types.PageToken next_page_token = 2;
+  optional string next_page_token = 2;
 }
 
 // Sign a message with the node's secret key.

ldk-server-grpc/src/proto/types.proto

@@ -880,12 +880,6 @@ message AwaitingThresholdConfirmations {
   uint64 amount_satoshis = 5;
 }
 
-// Token used to determine start of next page in paginated APIs.
-message PageToken {
-  string token = 1;
-  int64 index = 2;
-}
-
 message Bolt11InvoiceDescription {
   oneof kind {
     string direct = 1;

ldk-server-grpc/src/types.rs

@@ -1087,18 +1087,6 @@ pub struct AwaitingThresholdConfirmations {
 	#[prost(uint64, tag = "5")]
 	pub amount_satoshis: u64,
 }
-/// Token used to determine start of next page in paginated APIs.
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
-#[cfg_attr(feature = "serde", serde(default))]
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct PageToken {
-	#[prost(string, tag = "1")]
-	pub token: ::prost::alloc::string::String,
-	#[prost(int64, tag = "2")]
-	pub index: i64,
-}
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
 #[cfg_attr(feature = "serde", serde(default))]

ldk-server-mcp/src/tools/schema.rs

@@ -132,13 +132,8 @@ fn channel_config_schema() -> Value {
 
 fn page_token_schema() -> Value {
 	json!({
-		"type": "object",
-		"description": "Pagination token from a previous response",
-		"properties": {
-			"token": { "type": "string" },
-			"index": { "type": "integer" }
-		},
-		"required": ["token", "index"]
+		"type": "string",
+		"description": "Opaque pagination token from a previous response"
 	})
 }