feat!: add abitrary persistence

With the following approach we can accomplish two things at once:
1. Make it easy to add new Rust backends without sacrifing performance
2. Allow for arbitrary persistence accross the FFI

To accomplish this we can differentiate between a native backend and a
foreign backend. The foreign backend interacts with the FFI `ChangeSet`,
whereas the native backend can just use the Rust changeset directly.
Whenever a new backend is introduced in Rust, a new enum variant may
simply be added to the `PersistenceType`. To build a Sqlite backend, or
a foreign backend, the user will use the `Persister` structure.

Abitrary persistence is allowed by implementing the `Persistence`
trait. This was accomplished with no changes to #756.

I hope 1. motivates this change thoroughly, as we expect BDK will add
support for new backends in the future. I am also interested in the
applications of 2., where a user might be able to do encrypted and cloud
storage.

Author: rustaceanrob

bdk-ffi/src/bdk.udl

@@ -197,7 +197,7 @@ interface ParseAmountError {
 
 [Error]
 interface PersistenceError {
-  Write(string error_message);
+  Reason(string error_message);
 };
 
 [Error]

bdk-ffi/src/error.rs

@@ -564,8 +564,8 @@ pub enum ParseAmountError {
 
 #[derive(Debug, thiserror::Error)]
 pub enum PersistenceError {
-    #[error("writing to persistence error: {error_message}")]
-    Write { error_message: String },
+    #[error("persistence error: {error_message}")]
+    Reason { error_message: String },
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -745,12 +745,6 @@ pub enum SignerError {
     Psbt { error_message: String },
 }
 
-#[derive(Debug, thiserror::Error, uniffi::Error)]
-pub enum SqliteError {
-    #[error("sqlite error: {rusqlite_error}")]
-    Sqlite { rusqlite_error: String },
-}
-
 #[derive(Debug, thiserror::Error)]
 pub enum TransactionError {
     #[error("io error")]
@@ -1008,6 +1002,23 @@ impl From<PushBytesError> for CreateTxError {
     }
 }
 
+impl From<BdkCreateWithPersistError<PersistenceError>> for CreateWithPersistError {
+    fn from(error: BdkCreateWithPersistError<PersistenceError>) -> Self {
+        match error {
+            BdkCreateWithPersistError::Persist(e) => CreateWithPersistError::Persist {
+                error_message: e.to_string(),
+            },
+            BdkCreateWithPersistError::Descriptor(e) => CreateWithPersistError::Descriptor {
+                error_message: e.to_string(),
+            },
+            // Objects cannot currently be used in enumerations
+            BdkCreateWithPersistError::DataAlreadyExists(_e) => {
+                CreateWithPersistError::DataAlreadyExists
+            }
+        }
+    }
+}
+
 impl From<BdkCreateWithPersistError<chain::rusqlite::Error>> for CreateWithPersistError {
     fn from(error: BdkCreateWithPersistError<chain::rusqlite::Error>) -> Self {
         match error {
@@ -1251,6 +1262,14 @@ impl From<BdkLoadWithPersistError<chain::rusqlite::Error>> for LoadWithPersistEr
     }
 }
 
+impl From<BdkSqliteError> for PersistenceError {
+    fn from(error: BdkSqliteError) -> Self {
+        PersistenceError::Reason {
+            error_message: error.to_string(),
+        }
+    }
+}
+
 impl From<bdk_wallet::miniscript::Error> for MiniscriptError {
     fn from(error: bdk_wallet::miniscript::Error) -> Self {
         use bdk_wallet::miniscript::Error as BdkMiniscriptError;
@@ -1343,7 +1362,7 @@ impl From<BdkParseAmountError> for ParseAmountError {
 
 impl From<std::io::Error> for PersistenceError {
     fn from(error: std::io::Error) -> Self {
-        PersistenceError::Write {
+        PersistenceError::Reason {
             error_message: error.to_string(),
         }
     }
@@ -1513,14 +1532,6 @@ pub enum HashParseError {
     InvalidHash { len: u32 },
 }
 
-impl From<BdkSqliteError> for SqliteError {
-    fn from(error: BdkSqliteError) -> Self {
-        SqliteError::Sqlite {
-            rusqlite_error: error.to_string(),
-        }
-    }
-}
-
 impl From<bdk_kyoto::builder::SqlInitializationError> for CbfBuilderError {
     fn from(value: bdk_kyoto::builder::SqlInitializationError) -> Self {
         CbfBuilderError::DatabaseError {
@@ -1544,7 +1555,7 @@ mod test {
     use crate::error::SignerError;
     use crate::error::{
         Bip32Error, Bip39Error, CannotConnectError, DescriptorError, DescriptorKeyError,
-        ElectrumError, EsploraError, ExtractTxError, PersistenceError, PsbtError, PsbtParseError,
+        ElectrumError, EsploraError, ExtractTxError, PsbtError, PsbtParseError,
         RequestBuilderError, TransactionError, TxidParseError,
     };
 
@@ -1929,30 +1940,6 @@ mod test {
         }
     }
 
-    #[test]
-    fn test_persistence_error() {
-        let cases = vec![
-            (
-                std::io::Error::new(
-                    std::io::ErrorKind::Other,
-                    "unable to persist the new address",
-                )
-                .into(),
-                "writing to persistence error: unable to persist the new address",
-            ),
-            (
-                PersistenceError::Write {
-                    error_message: "failed to write to storage".to_string(),
-                },
-                "writing to persistence error: failed to write to storage",
-            ),
-        ];
-
-        for (error, expected_message) in cases {
-            assert_eq!(error.to_string(), expected_message);
-        }
-    }
-
     #[test]
     fn test_error_psbt() {
         let cases = vec![

bdk-ffi/src/store.rs

@@ -1,34 +1,86 @@
-use crate::error::SqliteError;
+use crate::error::PersistenceError;
+use crate::types::ChangeSet;
 
-use bdk_wallet::rusqlite::Connection as BdkConnection;
+use bdk_wallet::{rusqlite::Connection as BdkConnection, WalletPersister};
 
-use std::sync::Mutex;
-use std::sync::MutexGuard;
+use std::ops::DerefMut;
+use std::sync::{Arc, Mutex};
 
-/// A connection to a SQLite database.
+/// Definition of a wallet persistence implementation.
+#[uniffi::export]
+pub trait Persistence: Send + Sync {
+    /// Initialize the total aggregate `ChangeSet` for the underlying wallet.
+    fn initialize(&self) -> Result<ChangeSet, PersistenceError>;
+
+    /// Perist a `ChangeSet` to the total aggregate changeset of the wallet.
+    fn persist(&self, changeset: &ChangeSet) -> Result<(), PersistenceError>;
+}
+
+pub(crate) enum PersistenceType {
+    Foreign(Arc<dyn Persistence>),
+    Sql(Mutex<BdkConnection>),
+}
+
+/// Wallet backend implementations.
 #[derive(uniffi::Object)]
-pub struct Connection(Mutex<BdkConnection>);
+pub struct Persister {
+    pub(crate) inner: Mutex<PersistenceType>,
+}
 
 #[uniffi::export]
-impl Connection {
-    /// Open a new connection to a SQLite database. If a database does not exist at the path, one is
-    /// created.
+impl Persister {
+    /// Create a new Sqlite connection at the specified file path.
     #[uniffi::constructor]
-    pub fn new(path: String) -> Result<Self, SqliteError> {
-        let connection = BdkConnection::open(path)?;
-        Ok(Self(Mutex::new(connection)))
+    pub fn new_sqlite(path: String) -> Result<Self, PersistenceError> {
+        let conn = BdkConnection::open(path)?;
+        Ok(Self {
+            inner: PersistenceType::Sql(conn.into()).into(),
+        })
     }
 
-    /// Open a new connection to an in-memory SQLite database.
+    /// Create a new connection in memory.
     #[uniffi::constructor]
-    pub fn new_in_memory() -> Result<Self, SqliteError> {
-        let connection = BdkConnection::open_in_memory()?;
-        Ok(Self(Mutex::new(connection)))
+    pub fn new_in_memory() -> Result<Self, PersistenceError> {
+        let conn = BdkConnection::open_in_memory()?;
+        Ok(Self {
+            inner: PersistenceType::Sql(conn.into()).into(),
+        })
+    }
+
+    /// Use a native persistence layer.
+    #[uniffi::constructor]
+    pub fn custom(persistence: Arc<dyn Persistence>) -> Self {
+        Self {
+            inner: PersistenceType::Foreign(persistence).into(),
+        }
     }
 }
 
-impl Connection {
-    pub(crate) fn get_store(&self) -> MutexGuard<BdkConnection> {
-        self.0.lock().expect("must lock")
+impl WalletPersister for PersistenceType {
+    type Error = PersistenceError;
+
+    fn initialize(persister: &mut Self) -> Result<bdk_wallet::ChangeSet, Self::Error> {
+        match persister {
+            PersistenceType::Sql(ref conn) => {
+                let mut lock = conn.lock().unwrap();
+                let deref = lock.deref_mut();
+                Ok(BdkConnection::initialize(deref)?)
+            }
+            PersistenceType::Foreign(any) => any.initialize().map(|changeset| changeset.into()),
+        }
+    }
+
+    fn persist(persister: &mut Self, changeset: &bdk_wallet::ChangeSet) -> Result<(), Self::Error> {
+        match persister {
+            PersistenceType::Sql(ref conn) => {
+                let mut lock = conn.lock().unwrap();
+                let deref = lock.deref_mut();
+                Ok(BdkConnection::persist(deref, changeset)?)
+            }
+            PersistenceType::Foreign(any) => {
+                let ffi_changeset: ChangeSet = changeset.clone().into();
+                any.persist(&ffi_changeset)
+            }
+        }
     }
 }

bdk-ffi/src/tx_builder.rs

@@ -549,7 +549,7 @@ pub enum ChangeSpendPolicy {
 mod tests {
     use crate::bitcoin::{Amount, Script};
     use crate::{
-        descriptor::Descriptor, esplora::EsploraClient, store::Connection,
+        descriptor::Descriptor, esplora::EsploraClient, store::Persister,
         types::FullScanScriptInspector, wallet::Wallet,
     };
     use bdk_wallet::bitcoin::Network;
@@ -616,7 +616,7 @@ mod tests {
             Arc::new(Descriptor::new(external_descriptor, Network::Signet).unwrap()),
             Arc::new(Descriptor::new(internal_descriptor, Network::Signet).unwrap()),
             Network::Signet,
-            Arc::new(Connection::new_in_memory().unwrap()),
+            Arc::new(Persister::new_in_memory().unwrap()),
             25,
         )
         .unwrap();

bdk-ffi/src/wallet.rs

@@ -2,21 +2,20 @@ use crate::bitcoin::{Amount, FeeRate, OutPoint, Psbt, Script, Transaction, Txid}
 use crate::descriptor::Descriptor;
 use crate::error::{
     CalculateFeeError, CannotConnectError, CreateWithPersistError, DescriptorError,
-    LoadWithPersistError, SignerError, SqliteError, TxidParseError,
+    LoadWithPersistError, PersistenceError, SignerError, TxidParseError,
 };
-use crate::store::Connection;
+use crate::store::{PersistenceType, Persister};
 use crate::types::{
     AddressInfo, Balance, BlockId, CanonicalTx, FullScanRequestBuilder, KeychainAndIndex,
     LocalOutput, Policy, SentAndReceivedValues, SignOptions, SyncRequestBuilder, UnconfirmedTx,
     Update,
 };
 
 use bdk_wallet::bitcoin::Network;
-use bdk_wallet::rusqlite::Connection as BdkConnection;
 use bdk_wallet::signer::SignOptions as BdkSignOptions;
 use bdk_wallet::{KeychainKind, PersistedWallet, Wallet as BdkWallet};
 
-use std::borrow::BorrowMut;
+use std::ops::DerefMut;
 use std::sync::{Arc, Mutex, MutexGuard};
 
 /// A Bitcoin wallet.
@@ -33,7 +32,7 @@ use std::sync::{Arc, Mutex, MutexGuard};
 /// script pubkeys. See KeychainTxOutIndex::insert_descriptor() for more details.
 #[derive(uniffi::Object)]
 pub struct Wallet {
-    inner_mutex: Mutex<PersistedWallet<BdkConnection>>,
+    inner_mutex: Mutex<PersistedWallet<PersistenceType>>,
 }
 
 #[uniffi::export]
@@ -46,17 +45,17 @@ impl Wallet {
         descriptor: Arc<Descriptor>,
         change_descriptor: Arc<Descriptor>,
         network: Network,
-        connection: Arc<Connection>,
+        persistence: Arc<Persister>,
         lookahead: u32,
     ) -> Result<Self, CreateWithPersistError> {
         let descriptor = descriptor.to_string_with_secret();
         let change_descriptor = change_descriptor.to_string_with_secret();
-        let mut binding = connection.get_store();
-        let db: &mut BdkConnection = binding.borrow_mut();
+        let mut persist_lock = persistence.inner.lock().unwrap();
+        let deref = persist_lock.deref_mut();
         let params = BdkWallet::create(descriptor, change_descriptor)
             .network(network)
             .lookahead(lookahead);
-        let wallet: PersistedWallet<BdkConnection> = params.create_wallet(db)?;
+        let wallet: PersistedWallet<PersistenceType> = params.create_wallet(deref)?;
         Ok(Wallet {
             inner_mutex: Mutex::new(wallet),
         })
@@ -69,18 +68,21 @@ impl Wallet {
     pub fn load(
         descriptor: Arc<Descriptor>,
         change_descriptor: Arc<Descriptor>,
-        connection: Arc<Connection>,
+        persistence: Arc<Persister>,
     ) -> Result<Wallet, LoadWithPersistError> {
         let descriptor = descriptor.to_string_with_secret();
         let change_descriptor = change_descriptor.to_string_with_secret();
-        let mut binding = connection.get_store();
-        let db: &mut BdkConnection = binding.borrow_mut();
+        let mut persist_lock = persistence.inner.lock().unwrap();
+        let deref = persist_lock.deref_mut();
 
-        let wallet: PersistedWallet<BdkConnection> = BdkWallet::load()
+        let wallet: PersistedWallet<PersistenceType> = BdkWallet::load()
             .descriptor(KeychainKind::External, Some(descriptor))
             .descriptor(KeychainKind::Internal, Some(change_descriptor))
             .extract_keys()
-            .load_wallet(db)?
+            .load_wallet(deref)
+            .map_err(|e| LoadWithPersistError::Persist {
+                error_message: e.to_string(),
+            })?
             .ok_or(LoadWithPersistError::CouldNotLoad)?;
 
         Ok(Wallet {
@@ -404,13 +406,13 @@ impl Wallet {
     /// Returns whether any new changes were persisted.
     ///
     /// If the persister errors, the staged changes will not be cleared.
-    pub fn persist(&self, connection: Arc<Connection>) -> Result<bool, SqliteError> {
-        let mut binding = connection.get_store();
-        let db: &mut BdkConnection = binding.borrow_mut();
+    pub fn persist(&self, persistence: Arc<Persister>) -> Result<bool, PersistenceError> {
+        let mut persist_lock = persistence.inner.lock().unwrap();
+        let deref = persist_lock.deref_mut();
         self.get_wallet()
-            .persist(db)
-            .map_err(|e| SqliteError::Sqlite {
-                rusqlite_error: e.to_string(),
+            .persist(deref)
+            .map_err(|e| PersistenceError::Reason {
+                error_message: e.to_string(),
             })
     }
 
@@ -421,7 +423,7 @@ impl Wallet {
 }
 
 impl Wallet {
-    pub(crate) fn get_wallet(&self) -> MutexGuard<PersistedWallet<BdkConnection>> {
+    pub(crate) fn get_wallet(&self) -> MutexGuard<PersistedWallet<PersistenceType>> {
         self.inner_mutex.lock().expect("wallet")
     }
 }

bdk-jvm/lib/src/test/kotlin/org/bitcoindevkit/LiveElectrumClientTest.kt

@@ -18,7 +18,7 @@ class LiveElectrumClientTest {
 
     @Test
     fun testSyncedBalance() {
-        var conn: Connection = Connection.newInMemory()
+        var conn: Persister = Persister.newInMemory()
         val wallet: Wallet = Wallet(descriptor, changeDescriptor, Network.SIGNET, conn)
         val electrumClient: ElectrumClient = ElectrumClient(SIGNET_ELECTRUM_URL)
         val fullScanRequest: FullScanRequest = wallet.startFullScan().build()

bdk-jvm/lib/src/test/kotlin/org/bitcoindevkit/LiveKyotoTest.kt

@@ -28,7 +28,7 @@ class LiveKyotoTest {
 
     @Test
     fun testKyoto() {
-        val conn: Connection = Connection.newInMemory()
+        val conn: Persister = Persister.newInMemory()
         val wallet: Wallet = Wallet(descriptor, changeDescriptor, Network.SIGNET, conn)
         val peers = listOf(peer)
         runBlocking {