From 94ef8f41ac3c44fa8bad770b74a090928e6fd243 Mon Sep 17 00:00:00 2001 From: Omer Tuchfeld Date: Mon, 22 Dec 2025 18:15:33 +0100 Subject: [PATCH 1/3] Detect CBOR and fail gracefully Kubernetes started using CBOR in etcd values https://github.com/kubernetes/enhancements/issues/4222, for now we don't know how to handle it but we should fail gracefully. Further commits will add CBOR support to recert. --- src/etcd_encoding.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/etcd_encoding.rs b/src/etcd_encoding.rs index 232965c0..abaf2dc6 100644 --- a/src/etcd_encoding.rs +++ b/src/etcd_encoding.rs @@ -65,6 +65,10 @@ k8s_type!(OAuthClientWithMeta, OAuthClient); pub(crate) async fn decode(data: &[u8]) -> Result> { if !data.starts_with("k8s\x00".as_bytes()) { + // k8s uses CBOR with the self-describing tag 55799, we can use its bytes to detect CBOR + if data.starts_with([0xd9, 0xd9, 0xf7].as_ref()) { + bail!("CBOR encoding is not supported"); + } return Ok(data.to_vec()); } From 728312cc5dae1e9ce2663f8862a9dd29c23b36c3 Mon Sep 17 00:00:00 2001 From: Omer Tuchfeld Date: Mon, 22 Dec 2025 18:47:38 +0100 Subject: [PATCH 2/3] Parse CBOR into JSON If we encounter CBOR data (identified by the self-describing tag), we parse it and convert it internally to JSON. For now, we don't support writing CBOR back, we just write JSON back, which should work, but it means the cluster will lose its CBOR representation. In the next commits, we'll add the ability to write CBOR back as well. --- .cargo/config.toml | 5 -- Cargo.lock | 77 ++++++++++++++++++++++++++--- Cargo.toml | 1 + src/etcd_encoding.rs | 8 ++- src/etcd_encoding/k8s_cbor.rs | 92 +++++++++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 12 deletions(-) delete mode 100644 .cargo/config.toml create mode 100644 src/etcd_encoding/k8s_cbor.rs diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 02369289..00000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[source.crates-io] -replace-with = "vendored-sources" - -[source.vendored-sources] -directory = "vendor" diff --git a/Cargo.lock b/Cargo.lock index 1e09a9ea..e8716a71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,33 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -424,6 +451,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.5.3" @@ -851,6 +884,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1467,9 +1511,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -1530,9 +1574,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -1578,6 +1622,7 @@ dependencies = [ "bcder", "bytes", "chrono", + "ciborium", "clap", "clio", "data-url", @@ -2028,9 +2073,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.38" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -2667,6 +2712,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zeroize" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 712e36d6..91b0c7be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ aes-gcm = "0.10.3" dyn-clone = "1.0.17" hex = "0.4.3" serde_with = { version = "3.11.0", features = ["base64"] } +ciborium = "0.2.2" [build-dependencies] prost-build = "0.12.1" diff --git a/src/etcd_encoding.rs b/src/etcd_encoding.rs index abaf2dc6..d72bd6ca 100644 --- a/src/etcd_encoding.rs +++ b/src/etcd_encoding.rs @@ -63,12 +63,18 @@ k8s_type!(ValidatingWebhookConfigurationWithMeta, ValidatingWebhookConfiguration k8s_type!(MutatingWebhookConfigurationWithMeta, MutatingWebhookConfiguration); k8s_type!(OAuthClientWithMeta, OAuthClient); +mod k8s_cbor; + pub(crate) async fn decode(data: &[u8]) -> Result> { if !data.starts_with("k8s\x00".as_bytes()) { // k8s uses CBOR with the self-describing tag 55799, we can use its bytes to detect CBOR if data.starts_with([0xd9, 0xd9, 0xf7].as_ref()) { - bail!("CBOR encoding is not supported"); + // It's CBOR, just convert to JSON + let json_value = k8s_cbor::k8s_cbor_bytes_to_json(data).context("converting CBOR to JSON")?; + return Ok(serde_json::to_vec(&json_value)?); } + + // Not CBOR, not protobuf, it's probably just raw JSON, return as-is return Ok(data.to_vec()); } diff --git a/src/etcd_encoding/k8s_cbor.rs b/src/etcd_encoding/k8s_cbor.rs new file mode 100644 index 00000000..4e451f6d --- /dev/null +++ b/src/etcd_encoding/k8s_cbor.rs @@ -0,0 +1,92 @@ +use anyhow::{bail, Context, Result}; +use ciborium::value::Value as CborValue; +use serde_json::{value::Number as JsonNumber, Value as JsonValue}; + +const SELF_DESCRIBING_CBOR_TAG: u64 = 55799; + +fn cbor_to_json(cbor: CborValue) -> Result { + Ok(match cbor { + CborValue::Null => JsonValue::Null, + CborValue::Bool(boolean) => JsonValue::Bool(boolean), + CborValue::Text(string) => JsonValue::String(string), + CborValue::Integer(int) => JsonValue::Number({ + let int: i128 = int.into(); + if let Ok(int) = u64::try_from(int) { + JsonNumber::from(int) + } else if let Ok(int) = i64::try_from(int) { + JsonNumber::from(int) + } else { + JsonNumber::from_f64(int as f64).context("Integer not JSON compatible")? + } + }), + CborValue::Float(float) => JsonValue::Number(JsonNumber::from_f64(float).context("Float not JSON compatible")?), + CborValue::Array(vec) => JsonValue::Array(vec.into_iter().map(cbor_to_json).collect::>>()?), + CborValue::Map(map) => JsonValue::Object(serde_json::Map::from_iter( + map.into_iter() + .map(|(k, v)| { + let key_str = match k { + CborValue::Bytes(bytes) => String::from_utf8(bytes).context("Invalid UTF-8 in CBOR map key")?, + CborValue::Text(text) => text, + _ => bail!("Unsupported CBOR map key type {:?}", k), + }; + Ok((key_str, cbor_to_json(v)?)) + }) + .collect::>>()?, + )), + // TODO: Handle proposed-encoding tags for CBOR bytes? https://github.com/kubernetes/kubernetes/pull/125419 + // It seems that in a typical k8s cluster these are not used anywhere (secrets are + // protobuf, and they're pretty much the only place where raw bytes are used in + // values), so I don't have an example to test that implementation on. For now we will + // crash on unhandled tags below to be safe. + CborValue::Bytes(vec) => JsonValue::String(String::from_utf8(vec).context("Invalid UTF-8 in CBOR bytes")?), + CborValue::Tag(value, _tag) => unimplemented!("Unsupported CBOR tag {:?}", value), + _ => unimplemented!("Unsupported CBOR type {:?}", cbor), + }) +} + +fn json_to_cbor(json: JsonValue) -> Result { + Ok(match json { + JsonValue::Null => CborValue::Null, + JsonValue::Bool(boolean) => CborValue::Bool(boolean), + JsonValue::String(string) => CborValue::Bytes(string.into_bytes()), + JsonValue::Number(number) => { + if let Some(int) = number.as_i64() { + CborValue::Integer(int.into()) + } else if let Some(uint) = number.as_u64() { + CborValue::Integer(uint.into()) + } else if let Some(float) = number.as_f64() { + CborValue::Float(float) + } else { + bail!("Unsupported number type") + } + } + JsonValue::Array(arr) => CborValue::Array(arr.into_iter().map(json_to_cbor).collect::>>()?), + JsonValue::Object(map) => { + // Fallback for regular JSON objects (shouldn't happen in our flow) + let map_entries: Vec<(CborValue, CborValue)> = map + .into_iter() + .map(|(k, v)| Ok((CborValue::Bytes(k.into_bytes()), json_to_cbor(v)?))) + .collect::>>()?; + CborValue::Map(map_entries) + } + }) +} + +pub(crate) fn k8s_cbor_bytes_to_json(cbor_bytes: &[u8]) -> Result { + let v: CborValue = ciborium::de::from_reader(cbor_bytes)?; + + let (v, had_self_describing_tag) = match v { + CborValue::Tag(value, contents) => match value { + SELF_DESCRIBING_CBOR_TAG => { + // Self-describing CBOR tag, unwrap the contents + (*contents, true) + } + _ => panic!("Unsupported CBOR tag {}", value), + }, + // We expected a self-describing CBOR tag at the root. Of course we could just proceed + // as is (since it's just raw CBOR) but it's a bit fishy, so just bail + _ => bail!("CBOR data that does not start with self-describing tag is not supported"), + }; + + cbor_to_json(v) +} From 8f7bc5162c640434b32d3cf6c2872d330bff524b Mon Sep 17 00:00:00 2001 From: Omer Tuchfeld Date: Mon, 22 Dec 2025 21:44:58 +0100 Subject: [PATCH 3/3] Add CBOR support for etcd encoding Up until now we would write CBOR values back as etcd, after this commit we preserve the original encoding, so CBOR values are written back as CBOR. This required tracking the encoding type in the in-memory etcd cache, and modifying the encode function to handle CBOR. --- src/cluster_crypto/cert_key_pair.rs | 2 +- src/cluster_crypto/distributed_jwt.rs | 2 +- src/cluster_crypto/distributed_private_key.rs | 2 +- src/cluster_crypto/distributed_public_key.rs | 2 +- src/etcd_encoding.rs | 28 ++++++++++++++---- src/etcd_encoding/k8s_cbor.rs | 12 ++++++++ src/k8s_etcd.rs | 29 ++++++++++++------- .../encryption_config/etcd_rename.rs | 2 +- .../hostname_rename/etcd_rename.rs | 2 +- 9 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/cluster_crypto/cert_key_pair.rs b/src/cluster_crypto/cert_key_pair.rs index 37575d44..f2f147b0 100644 --- a/src/cluster_crypto/cert_key_pair.rs +++ b/src/cluster_crypto/cert_key_pair.rs @@ -357,7 +357,7 @@ impl CertKeyPair { .as_bytes() .to_vec(), ) - .await; + .await.context("putting in etcd")?; Ok(()) } diff --git a/src/cluster_crypto/distributed_jwt.rs b/src/cluster_crypto/distributed_jwt.rs index bd899bb6..467223e7 100644 --- a/src/cluster_crypto/distributed_jwt.rs +++ b/src/cluster_crypto/distributed_jwt.rs @@ -86,7 +86,7 @@ impl DistributedJwt { &k8slocation.resource_location.as_etcd_key(), serde_json::to_string(&resource)?.as_bytes().to_vec(), ) - .await; + .await.context("putting in etcd")?; Ok(()) } diff --git a/src/cluster_crypto/distributed_private_key.rs b/src/cluster_crypto/distributed_private_key.rs index 9166491a..c7773677 100644 --- a/src/cluster_crypto/distributed_private_key.rs +++ b/src/cluster_crypto/distributed_private_key.rs @@ -96,7 +96,7 @@ impl DistributedPrivateKey { .as_bytes() .to_vec(), ) - .await; + .await.context("putting in etcd")?; Ok(()) } diff --git a/src/cluster_crypto/distributed_public_key.rs b/src/cluster_crypto/distributed_public_key.rs index bc378654..dfec490a 100644 --- a/src/cluster_crypto/distributed_public_key.rs +++ b/src/cluster_crypto/distributed_public_key.rs @@ -129,7 +129,7 @@ impl DistributedPublicKey { .as_bytes() .to_vec(), ) - .await; + .await.context("putting in etcd")?; Ok(()) } diff --git a/src/etcd_encoding.rs b/src/etcd_encoding.rs index d72bd6ca..c9660484 100644 --- a/src/etcd_encoding.rs +++ b/src/etcd_encoding.rs @@ -65,17 +65,24 @@ k8s_type!(OAuthClientWithMeta, OAuthClient); mod k8s_cbor; -pub(crate) async fn decode(data: &[u8]) -> Result> { +#[derive(Clone)] +pub(crate) enum Encoding { + Protobuf, + Cbor, + Json, +} + +pub(crate) async fn decode(data: &[u8]) -> Result<(Vec, Encoding)> { if !data.starts_with("k8s\x00".as_bytes()) { // k8s uses CBOR with the self-describing tag 55799, we can use its bytes to detect CBOR if data.starts_with([0xd9, 0xd9, 0xf7].as_ref()) { // It's CBOR, just convert to JSON let json_value = k8s_cbor::k8s_cbor_bytes_to_json(data).context("converting CBOR to JSON")?; - return Ok(serde_json::to_vec(&json_value)?); + return Ok((serde_json::to_vec(&json_value)?, Encoding::Cbor)); } // Not CBOR, not protobuf, it's probably just raw JSON, return as-is - return Ok(data.to_vec()); + return Ok((data.to_vec(), Encoding::Json)); } let data = &data[4..]; @@ -89,7 +96,7 @@ pub(crate) async fn decode(data: &[u8]) -> Result> { .context("missing kind")? .as_str(); - Ok(match kind { + let decoded_data = match kind { "Route" => serde_json::to_vec(&RouteWithMeta::try_from(unknown)?)?, "Deployment" => serde_json::to_vec(&DeploymentWithMeta::try_from(unknown)?)?, "ControllerRevision" => serde_json::to_vec(&ControllerRevisionWithMeta::try_from(unknown)?)?, @@ -105,11 +112,20 @@ pub(crate) async fn decode(data: &[u8]) -> Result> { "MutatingWebhookConfiguration" => serde_json::to_vec(&MutatingWebhookConfigurationWithMeta::try_from(unknown)?)?, "OAuthClient" => serde_json::to_vec(&OAuthClientWithMeta::try_from(unknown)?)?, _ => bail!("unknown kind {}", kind), - }) + }; + + Ok((decoded_data, Encoding::Protobuf)) } -pub(crate) async fn encode(data: &[u8]) -> Result> { +pub(crate) async fn encode(data: &[u8], encoding: Encoding) -> Result> { let value: Value = serde_json::from_slice(data)?; + + if matches!(encoding, Encoding::Cbor) { + return k8s_cbor::json_to_k8s_cbor_bytes(value).context("converting JSON to CBOR"); + } + + // If kind is a known protobuf kind, write it back as protobuf, otherwise return raw JSON + // TODO: Just look at the new encoding param? let kind = value .pointer("/kind") .context("missing kind")? diff --git a/src/etcd_encoding/k8s_cbor.rs b/src/etcd_encoding/k8s_cbor.rs index 4e451f6d..7beeb37d 100644 --- a/src/etcd_encoding/k8s_cbor.rs +++ b/src/etcd_encoding/k8s_cbor.rs @@ -90,3 +90,15 @@ pub(crate) fn k8s_cbor_bytes_to_json(cbor_bytes: &[u8]) -> Result { cbor_to_json(v) } + +pub(crate) fn json_to_k8s_cbor_bytes(json: JsonValue) -> Result> { + let cbor = json_to_cbor(json)?; + + // Put back the self-describing CBOR tag that we stripped + let tagged_cbor = CborValue::Tag(SELF_DESCRIBING_CBOR_TAG, Box::new(cbor)); + + let mut bytes = Vec::new(); + ciborium::ser::into_writer(&tagged_cbor, &mut bytes)?; + + Ok(bytes) +} diff --git a/src/k8s_etcd.rs b/src/k8s_etcd.rs index d84afdc5..0373a7c7 100644 --- a/src/k8s_etcd.rs +++ b/src/k8s_etcd.rs @@ -3,6 +3,7 @@ use crate::encrypt::ResourceTransformers; use crate::etcd_encoding; use anyhow::{bail, ensure, Context, Result}; use etcd_client::{Client as EtcdClient, GetOptions}; +use etcd_encoding::Encoding; use futures_util::future::join_all; use serde_json::Value; use std::collections::{HashMap, HashSet}; @@ -22,7 +23,7 @@ pub(crate) struct EtcdResult { /// have to go through etcd for every single edit. pub(crate) struct InMemoryK8sEtcd { pub(crate) etcd_client: Option>, - etcd_keyvalue_hashmap: Mutex>>, + etcd_keyvalue_hashmap: Mutex)>>, edited: Mutex>>, deleted_keys: Mutex>, decrypt_resource_transformers: Option, @@ -105,10 +106,10 @@ impl InMemoryK8sEtcd { continue; } let key = key.clone(); - let value = value.clone(); + let (encoding, value) = value.clone(); let etcd_client = Arc::clone(etcd_client); - let mut value = etcd_encoding::encode(value.as_slice()).await.context("encoding value")?; + let mut value = etcd_encoding::encode(value.as_slice(), encoding).await.context("encoding value")?; if let Some(resource_transformers) = &self.encrypt_resource_transformers { // https://github.com/kubernetes/apiserver/blob/3423727e46efe7dfa40dcdb1a9c5c5027b07303d/pkg/storage/value/transformer.go#L172 @@ -184,7 +185,7 @@ impl InMemoryK8sEtcd { { let hashmap = self.etcd_keyvalue_hashmap.lock().await; - if let Some(value) = hashmap.get(&key) { + if let Some((_encoding, value)) = hashmap.get(&key) { result.value.clone_from(value); return Ok(Some(result)); } @@ -195,7 +196,7 @@ impl InMemoryK8sEtcd { if let Some(value) = get_result.kvs().first() { let raw_etcd_value = value.value(); - let mut decoded_value = etcd_encoding::decode(raw_etcd_value).await.context("decoding value")?; + let (mut decoded_value, mut encoding) = etcd_encoding::decode(raw_etcd_value).await.context("decoding value")?; if let Some(resource_transformers) = &self.decrypt_resource_transformers { // https://github.com/kubernetes/apiserver/blob/3423727e46efe7dfa40dcdb1a9c5c5027b07303d/pkg/storage/value/transformer.go#L110 @@ -209,7 +210,7 @@ impl InMemoryK8sEtcd { .decrypt(key.to_string(), raw_etcd_value.to_vec()) .await .context("decrypting etcd value")?; - decoded_value = etcd_encoding::decode(&plaintext_value).await.context("decoding value")?; + (decoded_value, encoding) = etcd_encoding::decode(&plaintext_value).await.context("decoding value")?; break; } } @@ -219,7 +220,7 @@ impl InMemoryK8sEtcd { self.etcd_keyvalue_hashmap .lock() .await - .insert(key.to_string(), decoded_value.clone()); + .insert(key.to_string(), (encoding, decoded_value.clone())); result.value = decoded_value; return Ok(Some(result)); @@ -228,10 +229,18 @@ impl InMemoryK8sEtcd { Ok(None) } - pub(crate) async fn put(&self, key: &str, value: Vec) { - self.etcd_keyvalue_hashmap.lock().await.insert(key.to_string(), value.clone()); + pub(crate) async fn put(&self, key: &str, value: Vec) -> Result<()> { + let mut hashmap = self.etcd_keyvalue_hashmap.lock().await; + + // Only put if the key already exists in the cache, preserving the encoding + let (encoding, _) = hashmap.get(key).context(format!("key '{}' not found in cache", key))?; + let encoding = encoding.clone(); // Clone the encoding + hashmap.insert(key.to_string(), (encoding, value.clone())); + drop(hashmap); // Release the lock early + self.deleted_keys.lock().await.remove(key); self.edited.lock().await.insert(key.to_string(), value); + Ok(()) } pub(crate) async fn list_keys(&self, resource_kind: &str) -> Result> { @@ -337,6 +346,6 @@ pub(crate) async fn get_etcd_json(client: &InMemoryK8sEtcd, k8slocation: &K8sRes pub(crate) async fn put_etcd_yaml(client: &InMemoryK8sEtcd, k8slocation: &K8sResourceLocation, value: Value) -> Result<()> { client .put(&k8slocation.as_etcd_key(), serde_json::to_string(&value)?.as_bytes().into()) - .await; + .await.context("putting in etcd")?; Ok(()) } diff --git a/src/ocp_postprocess/encryption_config/etcd_rename.rs b/src/ocp_postprocess/encryption_config/etcd_rename.rs index e8d6c60e..4fde9122 100644 --- a/src/ocp_postprocess/encryption_config/etcd_rename.rs +++ b/src/ocp_postprocess/encryption_config/etcd_rename.rs @@ -177,7 +177,7 @@ async fn update_encryption_key(component: &str, etcd_client: &Arc, origina &(format!("/kubernetes.io/secrets/openshift-etcd/{new_secret_name}")), serde_json::to_string(&etcd_value).context("serializing value")?.as_bytes().to_vec(), ) - .await; + .await.context("putting in etcd")?; etcd_client.delete(&key).await.context(format!("deleting {}", key))?;