From c020e9d06c16f4d4ae0e09dbfe724e4af2ff929b Mon Sep 17 00:00:00 2001 From: Junji Takakura Date: Mon, 1 Sep 2025 20:55:02 +0900 Subject: [PATCH] feat: implement libSQL provider with execute and query interfaces Signed-off-by: Junji Takakura --- providers/provider-sqldb-libsql/.gitignore | 9 + providers/provider-sqldb-libsql/Cargo.toml | 22 + providers/provider-sqldb-libsql/README.md | 93 +++++ .../component/.gitignore | 5 + .../component/Cargo.toml | 12 + .../provider-sqldb-libsql/component/README.md | 11 + .../component/src/lib.rs | 93 +++++ .../component/wasmcloud.lock | 4 + .../component/wasmcloud.toml | 5 + .../component/wit/deps.lock | 4 + .../component/wit/deps.toml | 1 + .../component/wit/deps/libsql/provider.wit | 7 + .../component/wit/deps/libsql/query.wit | 43 ++ .../component/wit/deps/libsql/types.wit | 37 ++ .../component/wit/world.wit | 15 + providers/provider-sqldb-libsql/compose.yaml | 11 + .../provider-sqldb-libsql/src/bindings.rs | 38 ++ providers/provider-sqldb-libsql/src/config.rs | 99 +++++ providers/provider-sqldb-libsql/src/main.rs | 22 + .../provider-sqldb-libsql/src/provider.rs | 390 ++++++++++++++++++ providers/provider-sqldb-libsql/wadm.yaml | 35 ++ .../provider-sqldb-libsql/wasmcloud.lock | 4 + .../provider-sqldb-libsql/wasmcloud.toml | 8 + .../provider-sqldb-libsql/wit/provider.wit | 7 + providers/provider-sqldb-libsql/wit/query.wit | 43 ++ providers/provider-sqldb-libsql/wit/types.wit | 37 ++ 26 files changed, 1055 insertions(+) create mode 100644 providers/provider-sqldb-libsql/.gitignore create mode 100644 providers/provider-sqldb-libsql/Cargo.toml create mode 100644 providers/provider-sqldb-libsql/README.md create mode 100644 providers/provider-sqldb-libsql/component/.gitignore create mode 100644 providers/provider-sqldb-libsql/component/Cargo.toml create mode 100644 providers/provider-sqldb-libsql/component/README.md create mode 100644 providers/provider-sqldb-libsql/component/src/lib.rs create mode 100644 providers/provider-sqldb-libsql/component/wasmcloud.lock create mode 100644 providers/provider-sqldb-libsql/component/wasmcloud.toml create mode 100644 providers/provider-sqldb-libsql/component/wit/deps.lock create mode 100644 providers/provider-sqldb-libsql/component/wit/deps.toml create mode 100644 providers/provider-sqldb-libsql/component/wit/deps/libsql/provider.wit create mode 100644 providers/provider-sqldb-libsql/component/wit/deps/libsql/query.wit create mode 100644 providers/provider-sqldb-libsql/component/wit/deps/libsql/types.wit create mode 100644 providers/provider-sqldb-libsql/component/wit/world.wit create mode 100644 providers/provider-sqldb-libsql/compose.yaml create mode 100644 providers/provider-sqldb-libsql/src/bindings.rs create mode 100644 providers/provider-sqldb-libsql/src/config.rs create mode 100644 providers/provider-sqldb-libsql/src/main.rs create mode 100644 providers/provider-sqldb-libsql/src/provider.rs create mode 100644 providers/provider-sqldb-libsql/wadm.yaml create mode 100644 providers/provider-sqldb-libsql/wasmcloud.lock create mode 100644 providers/provider-sqldb-libsql/wasmcloud.toml create mode 100644 providers/provider-sqldb-libsql/wit/provider.wit create mode 100644 providers/provider-sqldb-libsql/wit/query.wit create mode 100644 providers/provider-sqldb-libsql/wit/types.wit diff --git a/providers/provider-sqldb-libsql/.gitignore b/providers/provider-sqldb-libsql/.gitignore new file mode 100644 index 0000000..9b4edc2 --- /dev/null +++ b/providers/provider-sqldb-libsql/.gitignore @@ -0,0 +1,9 @@ +# Rust build artifacts +target/ + +# Wash build artifacts +build/ +*.par.gz + +# libsql database files +data/ diff --git a/providers/provider-sqldb-libsql/Cargo.toml b/providers/provider-sqldb-libsql/Cargo.toml new file mode 100644 index 0000000..a27ae98 --- /dev/null +++ b/providers/provider-sqldb-libsql/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sqldb-libsql-provider" +version = "0.1.0" +edition = "2021" +description = """ +wasmCloud SQL database provider for libSQL. +""" + +[workspace] + +[badges.maintenance] +status = "actively-developed" + +[dependencies] +anyhow = "1" +deadpool-libsql = { version = "0.1.0" } +libsql = { version = "0.9" } +serde = { version = "1", features = ["derive"] } +tokio = { version = "1", features = ["full"] } +tracing = "0.1" +wasmcloud-provider-sdk = { version = "0.13.0", features = ["otel"] } +wit-bindgen-wrpc = "0.9.0" diff --git a/providers/provider-sqldb-libsql/README.md b/providers/provider-sqldb-libsql/README.md new file mode 100644 index 0000000..9c1fe5b --- /dev/null +++ b/providers/provider-sqldb-libsql/README.md @@ -0,0 +1,93 @@ +# ðŸŠķ SQL Database libSQL Provider + +This capability provider implements the [`wasmcloud:sqldb-libsql`][wasmcloud-sqldb-libsql-wit] WIT package, which enables SQL-driven database interaction with a [libSQL][libsql] database cluster. + +This provider handles concurrent component connections, and components which are linked to it should specify configuration at link time (see [the named configuration settings section](#-named-configuration-settings) for more details. + +Want to read all the functionality included the interface? [Start from `provider.wit`][provider-wit] to read what this provider can do, and work your way to [`query.wit`][query-wit] and [`types.wit`][types-wit]. + +Note that connections are local to a single provider, so multiple providers running on the same lattice will _not_ share connections automatically. + +[libsql]: https://docs.turso.tech/libsql +[wasmcloud-sqldb-libsql-wit]: https://github.com/jtakakura/wasmcloud-provider-sqldb-libsql/tree/main/wit +[provider-wit]: https://github.com/jtakakura/wasmcloud-provider-sqldb-libsql/blob/main/wit/provider.wit +[query-wit]: https://github.com/jtakakura/wasmcloud-provider-sqldb-libsql/blob/main/wit/query.wit +[types-wit]: https://github.com/jtakakura/wasmcloud-provider-sqldb-libsql/blob/main/wit/types.wit + +## 📑 Named configuration Settings + +As connection details are considered sensitive information, they should be specified via named configuration to the provider, and _specified_ via link definitions. +WADM files should not be checked into source control containing secrets. + +New named configuration can be specified by using `wash config put`. + +| Property | Example | Description | +| ----------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LIBSQL_URL` | `http://localhost:8080` | Remote libSQL Database URL | +| `LIBSQL_NAMESPACE` | `libsql` | Remote libSQL Database Namespace | +| `LIBSQL_POOL_SIZE` | `12` | Maximum size of the connection pool (configures [max_size](https://docs.rs/deadpool-libsql/0.1.0/deadpool_libsql/struct.PoolConfig.html#structfield.max_size)) | + +Once named configuration with the keys above is created, it can be referenced as `target_config` for a link to this provider. + +For example, the following WADM manifest fragment: + +```yaml +- name: querier + type: component + properties: + image: file://./build/sqldb_libsql_query_s.wasm + traits: + - type: spreadscaler + properties: + instances: 1 + - type: link + properties: + target: sqldb-libsql + namespace: wasmcloud + package: libsql + interfaces: [execute, query] + target_config: + - name: default-libsql +``` + +The `querier` component in the snippet above specifies a link to a `sqldb-libsql` target, with `target_config` that is only specifies `name` (no `properties`). + +> [!WARNING] +> While `LIBSQL_AUTH_TOKEN` can be specified as named configuration, it should be specified as a secret. +> +> In a future version, this will be required. + +## 🔐 Secret Settings + +While most values can be specified via named configuration, sensitive values like the `LIBSQL_AUTH_TOKEN` should be specified via _secrets_. + +New secrets be specified by using `wash secrets put`. + +| Property | Example | Description | +| ------------------- | ---------- | ------------------------- | +| `LIBSQL_AUTH_TOKEN` | `xxxxx.yyyyy.zzzzz` | Remote libSQL Database Auth Token | + +Once a secret has been created, it can be referenced in the link to the provider. + +For example, the following WADM manifest fragment: + +```yaml +- name: querier + type: component + properties: + image: file://./build/sqldb_libsql_query_s.wasm + traits: + - type: spreadscaler + properties: + instances: 1 + - type: link + properties: + target: sqldb-libsql + namespace: wasmcloud + package: libsql + interfaces: [execute, query] + target_secrets: + - name: default-libsql-secrets +``` + +The `querier` component in the snippet above specifies a link to a `sqldb-libsql` target, with `target_config` that is only specifies `name` (no `properties`). diff --git a/providers/provider-sqldb-libsql/component/.gitignore b/providers/provider-sqldb-libsql/component/.gitignore new file mode 100644 index 0000000..288220e --- /dev/null +++ b/providers/provider-sqldb-libsql/component/.gitignore @@ -0,0 +1,5 @@ +# Rust build artifacts +target/ + +# Wash build artifacts +build/ diff --git a/providers/provider-sqldb-libsql/component/Cargo.toml b/providers/provider-sqldb-libsql/component/Cargo.toml new file mode 100644 index 0000000..5f3b155 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "sqldb-postgres-query" +edition = "2021" +version = "0.1.0" + +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = "0.43.0" diff --git a/providers/provider-sqldb-libsql/component/README.md b/providers/provider-sqldb-libsql/component/README.md new file mode 100644 index 0000000..a7fb508 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/README.md @@ -0,0 +1,11 @@ +# Custom template test component + +This component is meant to test the [custom template capability provider](../) by an implementation of the interface on the component. + +## Build + +Use `wash build` to build this component. + +## Deploy + +Use the [wadm.yaml](../wadm.yaml) in the parent directory to deploy this component alongside the provider. diff --git a/providers/provider-sqldb-libsql/component/src/lib.rs b/providers/provider-sqldb-libsql/component/src/lib.rs new file mode 100644 index 0000000..018bbc5 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/src/lib.rs @@ -0,0 +1,93 @@ +wit_bindgen::generate!({ generate_all }); + +// NOTE: the imports below is generated by wit_bindgen::generate, due to the +// WIT definition(s) specified in `wit` +use wasmcloud::libsql::execute::{ + execute, execute_batch, execute_transactional_batch, last_insert_rowid, +}; +use wasmcloud::libsql::query::query; +use wasmcloud::libsql::types::LibsqlValue; + +// NOTE: the `Guest` trait corresponds to the export of the `invoke` interface, +// namespaced to the current WIT namespace & package ("wasmcloud:examples") +use exports::wasmcloud::examples::invoke::Guest; + +/// This struct must implement the all `export`ed functionality +/// in the WIT definition (see `wit/component.wit`) +struct QueryRunner; + +const CREATE_TABLE_QUERY: &str = r#" +CREATE TABLE IF NOT EXISTS example ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + description text NOT NULL, + created_at TEXT NOT NULL DEFAULT (DATETIME('now', 'localtime')) +); +"#; + +/// A basic insert query +const INSERT_QUERY: &str = r#" +INSERT INTO example (description) VALUES (?1); +"#; + +/// A batch insert query, which inserts multiple rows in a single query +const INSERT_QUERIES_1: &str = r#" +INSERT INTO example (description) VALUES ("batch inserted example row#1"); +INSERT INTO example (description) VALUES ("batch inserted example row#2"); +INSERT INTO example (description) VALUES ("batch inserted example row#3"); +"#; + +const INSERT_QUERIES_2: &str = r#" +INSERT INTO example (description) VALUES ("batch inserted example row#4"); +INSERT INTO example (description) VALUES ("batch inserted example row#5"); +INSERT INTO example (description) VALUES ("batch inserted example row#6"); +"#; + +/// A SELECT query, which takes the ID insert query, using Postgres `RETURNING` syntax, +/// which returns the contents of the row that was inserted +const SELECT_QUERY: &str = r#" +SELECT * FROM example WHERE id = ?1; +"#; + +impl Guest for QueryRunner { + fn call() -> String { + // First, ensure the right table is present + if let Err(e) = execute(CREATE_TABLE_QUERY, &[]) { + return format!("ERROR: failed to create table: {e}"); + }; + + // Insert multiple rows in a single batch + if let Err(e) = execute_batch(INSERT_QUERIES_1) { + return format!("ERROR: failed to batch insert rows: {e}"); + }; + + // Insert multiple rows in a single transactional batch + if let Err(e) = execute_transactional_batch(INSERT_QUERIES_2) { + return format!("ERROR: failed to batch insert rows: {e}"); + }; + + // Insert a row into the example + if let Err(e) = execute( + INSERT_QUERY, + &[LibsqlValue::Text("inserted example row!".into())], + ) { + return format!("ERROR: failed to insert row: {e}"); + }; + + // Get the last insert row ID + let last_insert_rowid = match last_insert_rowid() { + Ok(id) => id, + Err(e) => return format!("ERROR: failed to get last insert row ID: {e}"), + }; + + // Do an explicit SELECT for the row we just inserted, using the ID that was returned + // + // NOTE: normally you would not need this SELECT, thanks to RETURNING: + // https://www.postgresql.org/docs/current/dml-returning.html + match query(SELECT_QUERY, &[LibsqlValue::Integer(last_insert_rowid)]) { + Ok(rows) => format!("SUCCESS: inserted and manually retrieved new row:\n{rows:#?}"), + Err(e) => format!("ERROR: failed to retrieve inserted row: {e}"), + } + } +} + +export!(QueryRunner); diff --git a/providers/provider-sqldb-libsql/component/wasmcloud.lock b/providers/provider-sqldb-libsql/component/wasmcloud.lock new file mode 100644 index 0000000..2ad4aa8 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wasmcloud.lock @@ -0,0 +1,4 @@ +# This file is automatically generated. +# It is not intended for manual editing. +version = 1 +packages = [] diff --git a/providers/provider-sqldb-libsql/component/wasmcloud.toml b/providers/provider-sqldb-libsql/component/wasmcloud.toml new file mode 100644 index 0000000..67213d1 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wasmcloud.toml @@ -0,0 +1,5 @@ +name = "Custom template test component" +language = "rust" +type = "component" + +[component] \ No newline at end of file diff --git a/providers/provider-sqldb-libsql/component/wit/deps.lock b/providers/provider-sqldb-libsql/component/wit/deps.lock new file mode 100644 index 0000000..dea368d --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wit/deps.lock @@ -0,0 +1,4 @@ +[libsql] +path = "../../wit" +sha256 = "d1be02228dfd4544135590c8bf8b2bf9d62f99a1c0fda601a90c23831d8355f9" +sha512 = "72ee6d35767d1cb385f1453454cd0aa5bc20c6977c78555b7c8fb124f0379bace608b1ed8de51568903c80f0a10b023a3695ffbf8d429c6d115c2e1191008910" diff --git a/providers/provider-sqldb-libsql/component/wit/deps.toml b/providers/provider-sqldb-libsql/component/wit/deps.toml new file mode 100644 index 0000000..438329d --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wit/deps.toml @@ -0,0 +1 @@ +libsql = { path = "../../wit" } diff --git a/providers/provider-sqldb-libsql/component/wit/deps/libsql/provider.wit b/providers/provider-sqldb-libsql/component/wit/deps/libsql/provider.wit new file mode 100644 index 0000000..2dc6712 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wit/deps/libsql/provider.wit @@ -0,0 +1,7 @@ +package wasmcloud:libsql@0.1.0-draft; + +world provider-sqldb-libsql { + export execute; + export query; + export types; +} diff --git a/providers/provider-sqldb-libsql/component/wit/deps/libsql/query.wit b/providers/provider-sqldb-libsql/component/wit/deps/libsql/query.wit new file mode 100644 index 0000000..3b4eb63 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wit/deps/libsql/query.wit @@ -0,0 +1,43 @@ +package wasmcloud:libsql@0.1.0-draft; + +/// Interface for executing queries against a libSQL database +interface execute { + use types.{libsql-value, result-row, query-error}; + + /// Execute a query that does not return rows, such as an `INSERT`, `UPDATE`, or `DELETE` + /// + /// Queries *must* be parameterized, with named arguments in the form of `?`, for example: + /// + /// ``` + /// INSERT INTO users (email, username) VALUES (?1, ?2); + /// ``` + execute: func(query: string, params: list) -> result; + + /// Execute a batch set of statements + /// + execute-batch: func(query: string) -> result, query-error>; + + /// Execute a batch set of statements atomically in a transaction + /// + execute-transactional-batch: func(query: string) -> result, query-error>; + + /// Get the last inserted row ID, if the last query was an `INSERT` + /// + last-insert-rowid: func() -> result; +} + +/// Interface for querying a libSQL database +interface query { + use types.{libsql-value, result-row, query-error}; + + /// Query a libSQL database, leaving connection/session management + /// to the callee/implementer of this interface (normally a provider configured with connection credentials) + /// + /// Queries *must* be parameterized, with named arguments in the form of `?`, for example: + /// + /// ``` + /// SELECT email,username FROM users WHERE id=?1; + /// ``` + /// + query: func(query: string, params: list) -> result, query-error>; +} diff --git a/providers/provider-sqldb-libsql/component/wit/deps/libsql/types.wit b/providers/provider-sqldb-libsql/component/wit/deps/libsql/types.wit new file mode 100644 index 0000000..8875ca4 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wit/deps/libsql/types.wit @@ -0,0 +1,37 @@ +package wasmcloud:libsql@0.1.0-draft; + +/// Types used by components and providers of a SQLDB libSQL interface +interface types { + + /// Errors that occur while executing queries + variant query-error { + /// A completely unexpected error, specific to executing queries + unexpected(string), + } + + /// libSQL data values, usable as parameters or via queries + /// see: https://www.sqlite.org/datatype3.html + /// + variant libsql-value { + null, + + // Numeric + integer(s64), + real(f64), + + // String + text(string), + + // Binary + blob(list), + } + + record result-row-entry { + /// Name of the result column + column-name: string, + /// Value of the result column + value: libsql-value, + } + + type result-row = list; +} diff --git a/providers/provider-sqldb-libsql/component/wit/world.wit b/providers/provider-sqldb-libsql/component/wit/world.wit new file mode 100644 index 0000000..1621d36 --- /dev/null +++ b/providers/provider-sqldb-libsql/component/wit/world.wit @@ -0,0 +1,15 @@ +package wasmcloud:examples; + +/// Invoke a component and receive string output. Similar to wasi:cli/command.run, without args +/// +/// This enables the component to be used with `wash call` +interface invoke { + /// Invoke a component + call: func() -> string; +} + +world component { + import wasmcloud:libsql/execute@0.1.0-draft; + import wasmcloud:libsql/query@0.1.0-draft; + export invoke; +} diff --git a/providers/provider-sqldb-libsql/compose.yaml b/providers/provider-sqldb-libsql/compose.yaml new file mode 100644 index 0000000..a5a1efd --- /dev/null +++ b/providers/provider-sqldb-libsql/compose.yaml @@ -0,0 +1,11 @@ +services: + libsql: + image: ghcr.io/tursodatabase/libsql-server:latest + platform: linux/amd64 + ports: + - "8080:8080" + - "5001:5001" + environment: + - SQLD_NODE=primary + volumes: + - ./data/libsql:/var/lib/sqld diff --git a/providers/provider-sqldb-libsql/src/bindings.rs b/providers/provider-sqldb-libsql/src/bindings.rs new file mode 100644 index 0000000..e75d0b3 --- /dev/null +++ b/providers/provider-sqldb-libsql/src/bindings.rs @@ -0,0 +1,38 @@ +// Bindgen happens here +wit_bindgen_wrpc::generate!({ + with: { + "wasmcloud:libsql/types@0.1.0-draft": generate, + "wasmcloud:libsql/execute@0.1.0-draft": generate, + "wasmcloud:libsql/query@0.1.0-draft": generate, + }, +}); + +// Start bindgen-generated type imports +pub(crate) use exports::wasmcloud::libsql::types; +pub(crate) use exports::wasmcloud::libsql::{execute, query}; + +pub(crate) use types::LibsqlValue; + +impl From for libsql::Value { + fn from(value: LibsqlValue) -> Self { + match value { + LibsqlValue::Null => libsql::Value::Null, + LibsqlValue::Integer(i) => libsql::Value::Integer(i), + LibsqlValue::Real(f) => libsql::Value::Real(f), + LibsqlValue::Text(s) => libsql::Value::Text(s), + LibsqlValue::Blob(b) => libsql::Value::Blob(b.to_vec()), + } + } +} + +impl From for LibsqlValue { + fn from(value: libsql::Value) -> Self { + match value { + libsql::Value::Null => LibsqlValue::Null, + libsql::Value::Integer(i) => LibsqlValue::Integer(i), + libsql::Value::Real(f) => LibsqlValue::Real(f), + libsql::Value::Text(s) => LibsqlValue::Text(s), + libsql::Value::Blob(b) => LibsqlValue::Blob(b.into()), + } + } +} diff --git a/providers/provider-sqldb-libsql/src/config.rs b/providers/provider-sqldb-libsql/src/config.rs new file mode 100644 index 0000000..1e46c82 --- /dev/null +++ b/providers/provider-sqldb-libsql/src/config.rs @@ -0,0 +1,99 @@ +use tracing::warn; +use wasmcloud_provider_sdk::{core::secrets::SecretValue, LinkConfig}; + +/// Creation options for a libSQL connection +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ConnectionCreateOptions { + /// URL of the libSQL cluster to connect to + pub url: String, + /// Auth token used when accessing the libSQL cluster + pub auth_token: String, + /// Optional namespace to use for the connection + pub namespace: Option, + /// Optional connection pool size + pub pool_size: Option, +} + +impl From for deadpool_libsql::Config { + fn from(opts: ConnectionCreateOptions) -> Self { + let database = deadpool_libsql::config::Database::Remote(deadpool_libsql::config::Remote { + url: opts.url, + auth_token: opts.auth_token, + namespace: opts.namespace, + remote_encryption: None, + }); + + let mut config = deadpool_libsql::Config::new(database); + if let Some(pool_size) = opts.pool_size { + config.pool = deadpool_libsql::PoolConfig { + max_size: pool_size, + ..deadpool_libsql::PoolConfig::default() + }; + } + config + } +} + +/// Parse the options for libSQL configuration from a [`HashMap`], with a given prefix to the keys +/// +/// For example given a prefix like `EXAMPLE_`, and a Hashmap that contains an entry like ("EXAMPLE_HOST", "localhost"), +/// the parsed [`ConnectionCreateOptions`] would contain "localhost" as the host. +pub(crate) fn extract_prefixed_conn_config( + prefix: &str, + link_config: &LinkConfig, +) -> Option { + let LinkConfig { + config, secrets, .. + } = link_config; + + let keys = [ + format!("{prefix}URL"), + format!("{prefix}AUTH_TOKEN"), + format!("{prefix}NAMESPACE"), + format!("{prefix}POOL_SIZE"), + ]; + match keys + .iter() + .map(|k| { + // Prefer fetching from secrets, but fall back to config if not found + match (secrets.get(k).and_then(SecretValue::as_string), config.get(k)) { + (Some(s), Some(_)) => { + warn!("secret value [{k}] was found in secrets, but also exists in config. The value in secrets will be used."); + Some(s) + } + (Some(s), _) => Some(s), + // Offer a warning for the auth token, but other values are fine to be in config + (None, Some(c)) if k == &format!("{prefix}AUTH_TOKEN") => { + warn!("secret value [{k}] was not found in secrets, but exists in config. Prefer using secrets for sensitive values."); + Some(c.as_str()) + } + (None, Some(c)) => { + Some(c.as_str()) + } + (_, None) => None, + } + }) + .collect::>>()[..] + { + [Some(url), auth_token, namespace, pool_size] => + { + let pool_size = pool_size.and_then(|pool_size| { + pool_size.parse::().ok().or_else(|| { + warn!("invalid pool size value [{pool_size}], using default"); + None + }) + }); + + Some(ConnectionCreateOptions { + url: url.to_string(), + auth_token: auth_token.map(|s| s.to_string()).unwrap_or_default(), + namespace: namespace.map(|s| s.to_string()), + pool_size, + }) + } + _ => { + warn!("failed to find required keys in configuration: [{:?}]", keys); + None + } + } +} diff --git a/providers/provider-sqldb-libsql/src/main.rs b/providers/provider-sqldb-libsql/src/main.rs new file mode 100644 index 0000000..7e65e10 --- /dev/null +++ b/providers/provider-sqldb-libsql/src/main.rs @@ -0,0 +1,22 @@ +//! This provider is a template that's meant to inform developers how to build a custom capability provider. +//! +//! The implementation in `./provider.rs` uses the `wasmcloud-provider-sdk` to provide a scaffold +//! for building a capability provider with a custom interface. Take note of the documentation +//! comments in the code to understand how to build a capability provider. + +mod bindings; +mod config; +mod provider; + +use provider::LibsqlProvider; + +/// Capability providers are native executables, so the entrypoint is the same as any other Rust +/// binary, `main()`. Typically the `main` function is kept simple and the provider logic is +/// implemented in a separate module. Head to the `provider.rs` file to see the implementation of +/// the `BlankSlateProvider`. +#[tokio::main] +async fn main() -> anyhow::Result<()> { + LibsqlProvider::run().await?; + eprintln!("Custom template provider exiting"); + Ok(()) +} diff --git a/providers/provider-sqldb-libsql/src/provider.rs b/providers/provider-sqldb-libsql/src/provider.rs new file mode 100644 index 0000000..612ac6f --- /dev/null +++ b/providers/provider-sqldb-libsql/src/provider.rs @@ -0,0 +1,390 @@ +#![cfg(not(doctest))] + +//! SQL-powered database access provider implementing `wasmcloud:libsql` for connecting +//! to libSQL clusters. +//! +//! This implementation is multi-threaded and operations between different actors +//! use different connections and can run in parallel. +//! + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{Context as _, Result}; +use deadpool_libsql::Pool; +use libsql::params_from_iter; +use tokio::sync::RwLock; +use tracing::{error, instrument, warn}; + +use wasmcloud_provider_sdk::{ + get_connection, initialize_observability, propagate_trace_for_ctx, run_provider, + serve_provider_exports, Context, LinkConfig, LinkDeleteInfo, Provider, +}; + +use crate::bindings::types::{LibsqlValue, QueryError, ResultRow, ResultRowEntry}; +use crate::bindings::{execute, query, serve}; +use crate::config::{extract_prefixed_conn_config, ConnectionCreateOptions}; + +/// A unique identifier for a created connection +type SourceId = String; + +#[derive(Clone, Default)] +pub struct LibsqlProvider { + /// Database connections indexed by source ID name + connections: Arc>>, +} + +impl LibsqlProvider { + fn name() -> &'static str { + "sqldb-libsql-provider" + } + + /// Run [`LibsqlProvider`] as a wasmCloud provider + pub async fn run() -> anyhow::Result<()> { + initialize_observability!( + LibsqlProvider::name(), + std::env::var_os("PROVIDER_SQLDB_LIBSQL_FLAMEGRAPH_PATH") + ); + let provider = LibsqlProvider::default(); + let shutdown = run_provider(provider.clone(), LibsqlProvider::name()) + .await + .context("failed to run provider")?; + let connection = get_connection(); + let wrpc = connection + .get_wrpc_client(connection.provider_key()) + .await?; + serve_provider_exports(&wrpc, provider, shutdown, serve) + .await + .context("failed to serve provider exports") + } + + /// Create and store a connection pool, if not already present + async fn ensure_pool( + &self, + source_id: &str, + create_opts: ConnectionCreateOptions, + ) -> Result<()> { + // Exit early if a pool with the given source ID is already present + { + let connections = self.connections.read().await; + if connections.get(source_id).is_some() { + return Ok(()); + } + } + + // Build the new connection pool + let runtime = Some(deadpool_libsql::Runtime::Tokio1); + let config = deadpool_libsql::Config::from(create_opts); + let pool = config + .create_pool(runtime) + .await + .context("failed to create connection pool")?; + + // Save the newly created connection to the pool + let mut connections = self.connections.write().await; + connections.insert(source_id.into(), pool); + Ok(()) + } + + /// Execute a query + async fn do_execute( + &self, + source_id: &str, + query: &str, + params: Vec, + ) -> Result { + let connections = self.connections.read().await; + let pool = connections.get(source_id).ok_or_else(|| { + QueryError::Unexpected(format!( + "missing connection pool for source [{source_id}] while executing" + )) + })?; + + let connection = pool.get().await.map_err(|e| { + QueryError::Unexpected(format!("failed to build connection from pool: {e}")) + })?; + + let result = connection + .execute(query, params_from_iter(params)) + .await + .map_err(|e| QueryError::Unexpected(format!("failed to perform execute: {e}")))?; + + Ok(result) + } + + /// Execute a batch of queries + async fn do_execute_batch( + &self, + source_id: &str, + query: &str, + ) -> Result, QueryError> { + let connections = self.connections.read().await; + let pool = connections.get(source_id).ok_or_else(|| { + QueryError::Unexpected(format!( + "missing connection pool for source [{source_id}] while executing" + )) + })?; + + let connection = pool.get().await.map_err(|e| { + QueryError::Unexpected(format!("failed to build connection from pool: {e}")) + })?; + + let batch_rows = connection + .execute_batch(query) + .await + .map_err(|e| QueryError::Unexpected(format!("failed to perform execute: {e}")))?; + + // Convert rows to the expected format + Ok(batch_rows_to_result_rows(batch_rows).await) + } + + /// Execute a batch of queries in a transaction + async fn do_execute_transactional_batch( + &self, + source_id: &str, + query: &str, + ) -> Result, QueryError> { + let connections = self.connections.read().await; + let pool = connections.get(source_id).ok_or_else(|| { + QueryError::Unexpected(format!( + "missing connection pool for source [{source_id}] while executing transaction" + )) + })?; + + let connection = pool.get().await.map_err(|e| { + QueryError::Unexpected(format!("failed to build connection from pool: {e}")) + })?; + + let batch_rows = connection + .execute_transactional_batch(query) + .await + .map_err(|e| QueryError::Unexpected(format!("failed to perform execute: {e}")))?; + + // Convert rows to the expected format + Ok(batch_rows_to_result_rows(batch_rows).await) + } + + /// Perform a query + async fn do_query( + &self, + source_id: &str, + query: &str, + params: Vec, + ) -> Result, QueryError> { + let connections = self.connections.read().await; + let pool = connections.get(source_id).ok_or_else(|| { + QueryError::Unexpected(format!( + "missing connection pool for source [{source_id}] while querying" + )) + })?; + + let connection = pool.get().await.map_err(|e| { + QueryError::Unexpected(format!("failed to build connection from pool: {e}")) + })?; + + let rows = connection + .query(query, params_from_iter(params)) + .await + .map_err(|e| QueryError::Unexpected(format!("failed to perform query: {e}")))?; + + // Convert rows to the expected format + Ok(rows_to_result_rows(rows).await) + } + + async fn do_last_insert_rowid(&self, source_id: &str) -> Result { + let connections = self.connections.read().await; + let pool = connections.get(source_id).ok_or_else(|| { + QueryError::Unexpected(format!( + "missing connection pool for source [{source_id}] while getting last insert row ID" + )) + })?; + + let connection = pool.get().await.map_err(|e| { + QueryError::Unexpected(format!("failed to build connection from pool: {e}")) + })?; + + Ok(connection.last_insert_rowid()) + } +} + +impl Provider for LibsqlProvider { + /// Handle being linked to a source (likely a component) as a target + /// + /// Components are expected to provide references to named configuration via link definitions + /// which contain keys named `LIBSQL_*` detailing configuration for connecting to libSQL. + #[instrument(level = "debug", skip_all, fields(source_id))] + async fn receive_link_config_as_target( + &self, + link_config @ LinkConfig { source_id, .. }: LinkConfig<'_>, + ) -> anyhow::Result<()> { + // Attempt to parse a configuration from the map with the prefix LIBSQL_ + let Some(db_cfg) = extract_prefixed_conn_config("LIBSQL_", &link_config) else { + // If we failed to find a config on the link, then we + warn!(source_id, "no link-level DB configuration"); + return Ok(()); + }; + + // Create a pool if one isn't already present for this particular source + if let Err(error) = self.ensure_pool(source_id, db_cfg).await { + error!(?error, source_id, "failed to create connection"); + }; + + Ok(()) + } + + /// Handle notification that a link is dropped + /// + /// Generally we can release the resources (connections) associated with the source + #[instrument(level = "info", skip_all, fields(source_id = info.get_source_id()))] + async fn delete_link_as_target(&self, info: impl LinkDeleteInfo) -> anyhow::Result<()> { + let source_id = info.get_source_id(); + let mut connections = self.connections.write().await; + connections.remove(source_id); + drop(connections); + Ok(()) + } + + /// Handle shutdown request by closing all connections + #[instrument(level = "debug", skip_all)] + async fn shutdown(&self) -> anyhow::Result<()> { + let mut connections = self.connections.write().await; + connections.drain(); + Ok(()) + } +} + +impl execute::Handler> for LibsqlProvider { + #[instrument(level = "debug", skip_all, fields(query))] + async fn execute( + &self, + ctx: Option, + query: String, + params: Vec, + ) -> Result> { + propagate_trace_for_ctx!(ctx); + let Some(Context { + component: Some(source_id), + .. + }) = ctx + else { + return Ok(Err(QueryError::Unexpected( + "unexpectedly missing source ID".into(), + ))); + }; + + Ok(self.do_execute(&source_id, &query, params).await) + } + + #[instrument(level = "debug", skip_all, fields(query))] + async fn execute_batch( + &self, + ctx: Option, + query: String, + ) -> Result, QueryError>> { + propagate_trace_for_ctx!(ctx); + let Some(Context { + component: Some(source_id), + .. + }) = ctx + else { + return Ok(Err(QueryError::Unexpected( + "unexpectedly missing source ID".into(), + ))); + }; + + Ok(self.do_execute_batch(&source_id, &query).await) + } + + #[instrument(level = "debug", skip_all, fields(query))] + async fn execute_transactional_batch( + &self, + ctx: Option, + query: String, + ) -> Result, QueryError>> { + propagate_trace_for_ctx!(ctx); + let Some(Context { + component: Some(source_id), + .. + }) = ctx + else { + return Ok(Err(QueryError::Unexpected( + "unexpectedly missing source ID".into(), + ))); + }; + + Ok(self + .do_execute_transactional_batch(&source_id, &query) + .await) + } + + #[instrument(level = "debug", skip_all)] + async fn last_insert_rowid(&self, ctx: Option) -> Result> { + propagate_trace_for_ctx!(ctx); + let Some(Context { + component: Some(source_id), + .. + }) = ctx + else { + return Ok(Err(QueryError::Unexpected( + "unexpectedly missing source ID".into(), + ))); + }; + + Ok(self.do_last_insert_rowid(&source_id).await) + } +} + +// Implement the required Handler trait for LibsqlProvider using the generated bindings +impl query::Handler> for LibsqlProvider { + #[instrument(level = "debug", skip_all, fields(query))] + async fn query( + &self, + ctx: Option, + query: String, + params: Vec, + ) -> Result, QueryError>> { + propagate_trace_for_ctx!(ctx); + let Some(Context { + component: Some(source_id), + .. + }) = ctx + else { + return Ok(Err(QueryError::Unexpected( + "unexpectedly missing source ID".into(), + ))); + }; + + Ok(self.do_query(&source_id, &query, params).await) + } +} + +async fn batch_rows_to_result_rows(mut batch_rows: libsql::BatchRows) -> Vec { + let mut result_rows = Vec::new(); + while let Some(Some(rows)) = batch_rows.next_stmt_row() { + result_rows.extend(rows_to_result_rows(rows).await); + } + result_rows +} + +async fn rows_to_result_rows(mut rows: libsql::Rows) -> Vec { + let mut result_rows = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + result_rows.push(row_to_result_row(&row)); + } + result_rows +} + +fn row_to_result_row(row: &libsql::Row) -> ResultRow { + let mut result_row = ResultRow::default(); + let column_count = row.column_count(); + for i in 0..column_count { + let name = row.column_name(i).unwrap_or_default().to_string(); + let value = LibsqlValue::from(row.get_value(i).unwrap()); + let entry = ResultRowEntry { + column_name: name, + value, + }; + result_row.push(entry); + } + result_row +} diff --git a/providers/provider-sqldb-libsql/wadm.yaml b/providers/provider-sqldb-libsql/wadm.yaml new file mode 100644 index 0000000..1a1f3bd --- /dev/null +++ b/providers/provider-sqldb-libsql/wadm.yaml @@ -0,0 +1,35 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: rust-sqldb-libsql-query + annotations: + version: v0.1.0 + description: | + Demo WebAssembly component using the wasmCloud SQLDB libSQL provider via the wasmcloud:libsql WIT interface +spec: + components: + - name: querier + type: component + properties: + image: file://./component/build/sqldb_postgres_query.wasm + traits: + - type: spreadscaler + properties: + instances: 1 + # Link the component to the provider on wasmcloud:example/system-info + - type: link + properties: + target: + name: sqldb-libsql + config: + - name: default-libsql + namespace: wasmcloud + package: libsql + interfaces: [query, execute] + + - name: sqldb-libsql + type: capability + properties: + image: file://./build/sqldb-libsql-provider.par.gz + config: + - name: default-libsql diff --git a/providers/provider-sqldb-libsql/wasmcloud.lock b/providers/provider-sqldb-libsql/wasmcloud.lock new file mode 100644 index 0000000..2ad4aa8 --- /dev/null +++ b/providers/provider-sqldb-libsql/wasmcloud.lock @@ -0,0 +1,4 @@ +# This file is automatically generated. +# It is not intended for manual editing. +version = 1 +packages = [] diff --git a/providers/provider-sqldb-libsql/wasmcloud.toml b/providers/provider-sqldb-libsql/wasmcloud.toml new file mode 100644 index 0000000..9580caf --- /dev/null +++ b/providers/provider-sqldb-libsql/wasmcloud.toml @@ -0,0 +1,8 @@ +name = "SQLDB libSQL" +language = "rust" +type = "provider" +version = "0.1.0" + +[provider] +vendor = "wasmCloud" +bin_name = "sqldb-libsql-provider" diff --git a/providers/provider-sqldb-libsql/wit/provider.wit b/providers/provider-sqldb-libsql/wit/provider.wit new file mode 100644 index 0000000..2dc6712 --- /dev/null +++ b/providers/provider-sqldb-libsql/wit/provider.wit @@ -0,0 +1,7 @@ +package wasmcloud:libsql@0.1.0-draft; + +world provider-sqldb-libsql { + export execute; + export query; + export types; +} diff --git a/providers/provider-sqldb-libsql/wit/query.wit b/providers/provider-sqldb-libsql/wit/query.wit new file mode 100644 index 0000000..3b4eb63 --- /dev/null +++ b/providers/provider-sqldb-libsql/wit/query.wit @@ -0,0 +1,43 @@ +package wasmcloud:libsql@0.1.0-draft; + +/// Interface for executing queries against a libSQL database +interface execute { + use types.{libsql-value, result-row, query-error}; + + /// Execute a query that does not return rows, such as an `INSERT`, `UPDATE`, or `DELETE` + /// + /// Queries *must* be parameterized, with named arguments in the form of `?`, for example: + /// + /// ``` + /// INSERT INTO users (email, username) VALUES (?1, ?2); + /// ``` + execute: func(query: string, params: list) -> result; + + /// Execute a batch set of statements + /// + execute-batch: func(query: string) -> result, query-error>; + + /// Execute a batch set of statements atomically in a transaction + /// + execute-transactional-batch: func(query: string) -> result, query-error>; + + /// Get the last inserted row ID, if the last query was an `INSERT` + /// + last-insert-rowid: func() -> result; +} + +/// Interface for querying a libSQL database +interface query { + use types.{libsql-value, result-row, query-error}; + + /// Query a libSQL database, leaving connection/session management + /// to the callee/implementer of this interface (normally a provider configured with connection credentials) + /// + /// Queries *must* be parameterized, with named arguments in the form of `?`, for example: + /// + /// ``` + /// SELECT email,username FROM users WHERE id=?1; + /// ``` + /// + query: func(query: string, params: list) -> result, query-error>; +} diff --git a/providers/provider-sqldb-libsql/wit/types.wit b/providers/provider-sqldb-libsql/wit/types.wit new file mode 100644 index 0000000..8875ca4 --- /dev/null +++ b/providers/provider-sqldb-libsql/wit/types.wit @@ -0,0 +1,37 @@ +package wasmcloud:libsql@0.1.0-draft; + +/// Types used by components and providers of a SQLDB libSQL interface +interface types { + + /// Errors that occur while executing queries + variant query-error { + /// A completely unexpected error, specific to executing queries + unexpected(string), + } + + /// libSQL data values, usable as parameters or via queries + /// see: https://www.sqlite.org/datatype3.html + /// + variant libsql-value { + null, + + // Numeric + integer(s64), + real(f64), + + // String + text(string), + + // Binary + blob(list), + } + + record result-row-entry { + /// Name of the result column + column-name: string, + /// Value of the result column + value: libsql-value, + } + + type result-row = list; +}