diff --git a/.gitignore b/.gitignore index 13efc081e..e067394c2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,7 @@ .DS_Store +.vscode + image-rs/scripts/attestation-agent shell.nix diff --git a/image-rs/Cargo.toml b/image-rs/Cargo.toml index d599bad79..17c3fc00d 100644 --- a/image-rs/Cargo.toml +++ b/image-rs/Cargo.toml @@ -16,8 +16,9 @@ attestation_agent = { path = "../attestation-agent/lib", default-features = fals base64.workspace = true cfg-if = { workspace = true, optional = true } devicemapper = { version = "0.33.5", optional = true } +#devicemapper = { git="https://github.com/konsougiou/devicemapper-rs", rev="29d6dd5", optional = true} dircpy = { version = "0.3.12", optional = true } -flate2 = "1.0" +flate2 = "1.0.25" fs_extra = { version = "1.2.0", optional = true } futures = { version = "0.3.28", optional = true } futures-util = "0.3" @@ -41,7 +42,8 @@ sha2.workspace = true sigstore = { git = "https://github.com/sigstore/sigstore-rs.git", rev = "69e8f33", default-features = false, optional = true} strum.workspace = true strum_macros = "0.25" -tar = "0.4.37" +#tar = "0.4.37" +tar = "0.4.40" tokio.workspace = true tonic = { workspace = true, optional = true } ttrpc = { workspace = true, features = [ "async" ], optional = true } @@ -49,8 +51,10 @@ url = "2.2.2" walkdir = "2" zstd = "0.12" -nydus-api = { version = "0.3.0", optional = true} -nydus-service = { version = "0.3.0", features = ["coco"], optional = true} +nydus-api = { git="https://github.com/konsougiou/nydus.git", rev="440eee9", default-features = false, optional = true} +#nydus-api = { version = "0.3.0", optional = true} +nydus-service = { git="https://github.com/konsougiou/nydus.git", rev="440eee9", features = ["coco"], default-features = false, optional = true} +#nydus-service = { version = "0.3.0", features = ["coco"], optional = true} [build-dependencies] anyhow.workspace = true @@ -81,11 +85,13 @@ default = ["snapshot-overlayfs", "signature-cosign-rustls", "keywrap-grpc", "oci # This will be based on `ring` dependency kata-cc-rustls-tls = ["encryption-ring", "keywrap-ttrpc", "snapshot-overlayfs", "signature-cosign-rustls", "signature-simple", "getresource", "oci-distribution/rustls-tls"] + enclave-cc-eaakbc-rustls-tls = ["encryption-ring", "keywrap-native", "eaa-kbc", "snapshot-unionfs", "signature-simple", "getresource", "signature-cosign-rustls", "oci-distribution-rustls"] enclave-cc-cckbc-rustls-tls = ["encryption-ring", "keywrap-native", "cc-kbc-sgx", "snapshot-unionfs", "signature-simple", "getresource", "signature-cosign-rustls", "oci-distribution-rustls"] # This will be based on `openssl` dependency kata-cc-native-tls = ["encryption-openssl", "keywrap-ttrpc", "snapshot-overlayfs", "signature-cosign-native", "signature-simple", "getresource", "oci-distribution/native-tls"] +#kata-cc-native-tls = ["encryption-openssl", "keywrap-ttrpc", "signature-cosign-native", "signature-simple", "getresource", "oci-distribution/native-tls"] enclave-cc-eaakbc-native-tls = ["encryption-openssl", "keywrap-native", "eaa-kbc", "snapshot-unionfs", "signature-simple", "getresource", "signature-cosign-native", "oci-distribution-native"] enclave-cc-cckbc-native-tls = ["encryption-openssl", "keywrap-native", "cc-kbc-sgx", "snapshot-unionfs", "signature-simple", "getresource", "signature-cosign-native", "oci-distribution-native"] @@ -120,6 +126,6 @@ snapshot-unionfs = ["nix", "dircpy", "fs_extra"] getresource = [ "lazy_static", "cfg-if" ] -nydus = ["lazy_static", "nydus-api", "nydus-service"] +nydus = ["nix", "lazy_static", "nydus-api", "nydus-service"] verity = ["devicemapper"] diff --git a/image-rs/src/image.rs b/image-rs/src/image.rs index 1a747ae42..13e33e3f0 100644 --- a/image-rs/src/image.rs +++ b/image-rs/src/image.rs @@ -11,9 +11,16 @@ use oci_spec::image::{ImageConfiguration, Os}; use serde::Deserialize; use std::collections::{BTreeSet, HashMap}; use std::convert::TryFrom; -use std::path::Path; use std::sync::Arc; +use tokio::task; +use std::path::Path; +use std::fs::{self, File}; +use std::io::Write; +use std::process::Command; +use std::str; +use reqwest::blocking::Client; + use tokio::sync::Mutex; use crate::bundle::{create_runtime_config, BUNDLE_ROOTFS}; @@ -98,6 +105,9 @@ impl Default for ImageClient { // construct a default instance of `ImageClient` fn default() -> ImageClient { let config = ImageConfig::try_from(Path::new(CONFIGURATION_FILE_PATH)).unwrap_or_default(); + + //println!("KS-image-rs: Starting ImageClient with config: ({:?})", config); + let meta_store = MetaStore::try_from(Path::new(METAFILE)).unwrap_or_default(); #[allow(unused_mut)] @@ -114,6 +124,8 @@ impl Default for ImageClient { data_dir, std::sync::atomic::AtomicUsize::new(*overlay_index), ); + + //println!("KS-image-rs: Starting overlayfs snapshotter"); snapshots.insert( SnapshotType::Overlay, Box::new(overlayfs) as Box, @@ -146,6 +158,54 @@ impl Default for ImageClient { } } +fn dummy_prefetch() -> Result<(), Box> { + let client = Client::new(); + let blob_ids = ["ac2c9c7c25e992c7a0f1b6261112df95281324d8229541317f763dfaf01c7f30", "c737fc16374b9e9a352300146ab49de56f0068e42618fe2ebe3323d4069b7b89"]; + let cache_dir = "/opt/nydus/cache/"; + println!("KS: dummy pre-fetch"); + fs::create_dir_all(cache_dir)?; + + for &blob_id in &blob_ids { + println!("KS: pre fetching blob_id: {}", blob_id); + let url = format!("https://external-registry.coco-csg.com/v2/tf-serving-tinybert/blobs/sha256:{}", blob_id); + let response = client.get(&url).send()?; + + if response.status().is_success() { + let content = response.bytes()?; + + let cache_path = format!("{}{}", cache_dir, blob_id); + let mut file = File::create(cache_path)?; + file.write_all(&content)?; + } else { + eprintln!("KS Failed to fetch blob: {}", blob_id); + } + } + + let cmd = "ls /opt/nydus/cache/"; + let output = Command::new("sh") + .arg("-c") + .arg(cmd) + .output() + .expect("Failed to execute 'ls' command"); + + if output.status.success() { + let stdout = str::from_utf8(&output.stdout) + .unwrap_or("Failed to decode stdout as UTF-8"); + for line in stdout.split('\n') { + if !line.is_empty() { + println!("Blob: {}", line); + } + } + } else { + let stderr = str::from_utf8(&output.stderr) + .unwrap_or("Failed to decode stderr as UTF-8"); + eprintln!("Failed to execute '{}': {}", cmd, stderr); + } + + Ok(()) +} + + impl ImageClient { /// pull_image pulls an image with optional auth info and decrypt config /// and store the pulled data under user defined work_dir/layers. @@ -166,7 +226,20 @@ impl ImageClient { auth_info: &Option<&str>, decrypt_config: &Option<&str>, ) -> Result { + + // task::spawn_blocking(|| { + // if let Err(e) = dummy_prefetch() { + // eprintln!("Error occurred: {}", e); + // } + // }).await?; + + //let image_url = "external-registry.coco-csg.com/tf-serving-tinybert:unencrypted-nydus"; + //println!("KS-image-rs: pull_image called with image_url {:?}", image_url); + let image_url: &str = &image_url.replace("blob-cache", "unencrypted-nydus"); + println!("KS-image-rs: adjusted image_url {:?}", image_url); + let reference = Reference::try_from(image_url)?; + //let reference = Reference::try_from("external-registry.coco-csg.com/tf-serving-tinybert:unencrypted-nydus")?; // Try to get auth using input param. let auth = if let Some(auth_info) = auth_info { @@ -232,6 +305,7 @@ impl ImageClient { (false, true) => RegistryAuth::Anonymous, _ => auth.expect("unexpected uninitialized auth"), }; + //println!("KS-image-rs: Instantiating PullClient"); let mut client = PullClient::new( reference, @@ -239,8 +313,12 @@ impl ImageClient { &auth, self.config.max_concurrent_download, )?; + //println!("CSG-M4GIC: B3G1N: (KS-image-rs) Pull Manifest"); + let (image_manifest, image_digest, image_config) = client.pull_manifest().await?; + //println!("CSG-M4GIC: END: (KS-image-rs) Pull Manifest"); + let id = image_manifest.config.digest.clone(); let snapshot = match self.snapshots.get_mut(&self.config.default_snapshot) { @@ -255,6 +333,7 @@ impl ImageClient { #[cfg(feature = "nydus")] if utils::is_nydus_image(&image_manifest) { + println!("KS-image-rs: Nydus image detected"); { let m = self.meta_store.lock().await; if let Some(image_data) = &m.image_db.get(&id) { @@ -281,8 +360,8 @@ impl ImageClient { &image_digest, &image_config, )?; - - return self + //println!("CSG-M4GIC: B3G1N: (KS-image-rs) Nydus Image Pull"); + let ret = self .do_pull_image_with_nydus( &mut client, &mut image_data, @@ -291,18 +370,22 @@ impl ImageClient { bundle_dir, ) .await; + //println!("CSG-M4GIC: END: (KS-image-rs) Nydus Image Pull"); + return ret } // If image has already been populated, just create the bundle. { let m = self.meta_store.lock().await; if let Some(image_data) = &m.image_db.get(&id) { + //println!("KS-image-rs: meta_store already populated with ({:?})", image_data); return create_bundle(image_data, bundle_dir, snapshot); } } #[cfg(feature = "signature")] if self.config.security_validate { + println!("CSG-M4GIC: B3G1N: (KS-image-rs) Signature Validation"); crate::signature::allows_image( image_url, &image_digest, @@ -311,6 +394,7 @@ impl ImageClient { ) .await .map_err(|e| anyhow!("Security validate failed: {:?}", e))?; + println!("CSG-M4GIC: END: (KS-image-rs) Signature Validation"); } let (mut image_data, unique_layers, unique_diff_ids) = create_image_meta( @@ -322,6 +406,7 @@ impl ImageClient { )?; let unique_layers_len = unique_layers.len(); + //println!("CSG-M4GIC: B3G1N: Pull Layers ({:?})", image_url); let layer_metas = client .async_pull_layers( unique_layers, @@ -330,6 +415,7 @@ impl ImageClient { self.meta_store.clone(), ) .await?; + //println!("CSG-M4GIC: END: Pull Layers ({:?})", image_url); image_data.layer_metas = layer_metas; let layer_db: HashMap = image_data @@ -338,7 +424,12 @@ impl ImageClient { .map(|layer| (layer.compressed_digest.clone(), layer.clone())) .collect(); + // for (key, value) in layer_db.clone() { + // println!("KS-image-rs layer_db entry: {} => {:?}", key, value); + // } + self.meta_store.lock().await.layer_db.extend(layer_db); + if unique_layers_len != image_data.layer_metas.len() { bail!( " {} layers failed to pull", @@ -354,6 +445,12 @@ impl ImageClient { .image_db .insert(image_data.id.clone(), image_data.clone()); + let meta_store_lock = self.meta_store.lock().await; + // for (key, value) in meta_store_lock.image_db.iter() { + // println!("KS-image-rs image_db entry: {} => {:?}", key, value); + // } + + Ok(image_id) } @@ -373,6 +470,8 @@ impl ImageClient { bail!("Failed to get bootstrap id, diff_ids is empty"); }; + //println!("CSG-M4GIC: B3G1N: (KS-image-rs) Nydus Bootstrap Pull"); + let bootstrap = utils::get_nydus_bootstrap_desc(image_manifest) .ok_or_else(|| anyhow!("Faild to get bootstrap oci descriptor"))?; let layer_metas = client @@ -383,6 +482,9 @@ impl ImageClient { self.meta_store.clone(), ) .await?; + //println!("CSG-M4GIC: END: (KS-image-rs) Nydus Bootstrap Pull"); + + //println!("CSG-M4GIC: B3G1N: (KS-image-rs) Handle Bootstrap"); image_data.layer_metas = vec![layer_metas]; let layer_db: HashMap = image_data .layer_metas @@ -408,9 +510,10 @@ impl ImageClient { bail!( "default snapshot {} not found", &self.config.default_snapshot - ); + ); } }; + //println!("KS-image-rs: Starting nydus service"); let image_id = service::start_nydus_service( image_data, reference, @@ -427,6 +530,8 @@ impl ImageClient { .image_db .insert(image_data.id.clone(), image_data.clone()); + //println!("CSG-M4GIC: END: (KS-image-rs) Handle Bootstrap"); + Ok(image_id) } } diff --git a/image-rs/src/nydus/service.rs b/image-rs/src/nydus/service.rs index 3631301bf..bc277ca3f 100644 --- a/image-rs/src/nydus/service.rs +++ b/image-rs/src/nydus/service.rs @@ -50,6 +50,7 @@ pub async fn start_nydus_service( let work_dir_buf = work_dir.to_owned(); if nydus_config.is_fuse() { + //println!("KS-image-rs: fuse mode detected"); let fuse_config = nydus_config .get_fuse_config() .expect("Fuse configuration not found") @@ -69,6 +70,7 @@ pub async fn start_nydus_service( bail!("Failed to start nydus service, {:?}", e); }; } else if nydus_config.is_fscache() { + //println!("KS-image-rs: fscache mode detected"); let fscache_config = nydus_config .get_fscache_config() .expect("Fscache configuration not found") @@ -88,7 +90,6 @@ pub async fn start_nydus_service( } else { bootstrap_meta.compressed_digest.clone() }; - if let Err(e) = task::spawn_blocking(move || { process_fscache_daemon( id, @@ -167,6 +168,7 @@ pub fn process_fuse_daemon( reference.repository(), work_dir.join("cache"), ); + //println!("KS-image-rs: process_fuse_deamon called, with reference: {:?}, config ({:?}), bootstrap ({:?}), mountpoint ({:?}), workdir ({:?}) and fuse_config ({:?})", reference, config, bootstrap, mountpoint, work_dir, fuse_config); if !mountpoint.exists() { std::fs::create_dir_all(mountpoint)?; diff --git a/image-rs/src/nydus/utils.rs b/image-rs/src/nydus/utils.rs index cae497b22..5bf2a34c3 100644 --- a/image-rs/src/nydus/utils.rs +++ b/image-rs/src/nydus/utils.rs @@ -20,6 +20,8 @@ pub fn is_nydus_meta_layer(desc: &manifest::OciDescriptor) -> bool { } pub fn is_nydus_image(image_manifest: &manifest::OciImageManifest) -> bool { + + //println!("KS-image-rs: Checking if image is nydus"); get_nydus_bootstrap_desc(image_manifest).is_some() } @@ -32,9 +34,11 @@ pub fn get_nydus_bootstrap_desc( if is_nydus_meta_layer(desc) { Some(desc.clone()) } else { + println!("KS-image-rs: is nydus meta layers == false"); None } } else { + //println!("KS-image-rs: layers is empty. Not a nydus image"); None } } diff --git a/image-rs/src/pull.rs b/image-rs/src/pull.rs index b555b233b..309e99a63 100644 --- a/image-rs/src/pull.rs +++ b/image-rs/src/pull.rs @@ -100,20 +100,25 @@ impl<'a> PullClient<'a> { let ms = meta_store.clone(); async move { + let layer_digest = layer.digest.clone(); + //println!("CSG-M4GIC: B3G1N: Pull Single Layer ({:?})", layer_digest); let layer_reader = client .async_pull_blob(reference, &layer.digest) .await .map_err(|e| anyhow!("failed to async pull blob {}", e.to_string()))?; + //println!("CSG-M4GIC: END: Pull Single Layer ({:?})", layer_digest); - self.async_handle_layer( + //println!("CSG-M4GIC: B3G1N: Handle Single Layer ({:?})", layer_digest); + let tmp = self.async_handle_layer( layer, diff_ids[i].clone(), decrypt_config, layer_reader, ms, ) - .await - .map_err(|e| anyhow!("failed to handle layer: {:?}", e)) + .await; + //println!("CSG-M4GIC: END: Handle Single Layer ({:?})", layer_digest); + tmp.map_err(|e| anyhow!("failed to handle layer: {:?}", e)) } }) .buffer_unordered(self.max_concurrent_download) diff --git a/image-rs/src/verity/dmverity.rs b/image-rs/src/verity/dmverity.rs index d7c2e8847..30d086764 100644 --- a/image-rs/src/verity/dmverity.rs +++ b/image-rs/src/verity/dmverity.rs @@ -8,6 +8,10 @@ use devicemapper::{DevId, DmFlags, DmName, DmOptions, DM}; use serde::{Deserialize, Serialize}; use serde_json; use std::path::Path; +use std::str; +use std::process::Command; +use nix::unistd::{access, AccessFlags}; + /// Configuration information for DmVerity device. #[derive(Debug, Deserialize, Serialize)] @@ -129,6 +133,66 @@ impl TryFrom<&String> for DmVerityOption { } } +fn udev_running() -> bool { + const UDEV_SOCKET_PATH: &str = "/run/udev/control"; + matches!( + access(Path::new(UDEV_SOCKET_PATH), AccessFlags::F_OK), + Ok(()) + ) +} + +fn start_udev() { + let check_udev = Command::new("pgrep") + .args(&["-x", "systemd-udevd"]) + .output() + .expect("KS (image-rs) Failed to execute pgrep"); + + if check_udev.status.success() { + println!("KS (image-rs) udev daemon is already running."); + //println!("KS (image-rs) pgrep output: {}", str::from_utf8(&check_udev.stdout).unwrap()); + } else { + println!("KS (image-rs) udev daemon is not running."); + //println!("KS (image-rs) pgrep stderr: {}", str::from_utf8(&check_udev.stderr).unwrap()); + println!("KS (image-rs) Attempting to start udev..."); + + let cmds = [ + "/lib/systemd/systemd-udevd --daemon", + "udevadm trigger", + "udevadm settle", + ]; + for cmd in cmds.iter() { + let output = Command::new("sh") + .arg("-c") + .arg(cmd) + .output() + .expect("KS (image-rs) Failed to execute udev command"); + + if output.status.success() { + println!("KS (image-rs) Command '{}' executed successfully.", cmd); + } else { + eprintln!("KS (image-rs) Failed to execute '{}': {}", cmd, str::from_utf8(&output.stderr).unwrap()); + } + } + let check_udev = Command::new("pgrep") + .args(&["-x", "systemd-udevd"]) + .output() + .expect("KS (image-rs) Failed to exec pgrep"); + + if check_udev.status.success() { + println!("KS (image-rs) udev daemon sucesfully started."); + //println!("KS (image-rs) pgrep output: {}", str::from_utf8(&check_udev.stdout).unwrap()); + } else { + println!("KS (image-rs) udev daemon not started."); + //println!("KS (image-rs) pgrep stderr: {}", str::from_utf8(&check_udev.stderr).unwrap()); + } + } + if udev_running() { + println!("KS udev/control is accessible."); + } else { + println!("KS udev/control is NOT accessible."); + } +} + /// Creates a mapping with backed by data_device /// and using hash_device for in-kernel verification. /// It will return the verity block device Path "/dev/mapper/" @@ -137,6 +201,28 @@ pub fn create_verity_device( verity_option: &DmVerityOption, source_device_path: &Path, ) -> Result { + println!("CSG-M4GIC: B3G1N: (KS-image-rs) create_verity_device, path: ({:?}), option: ({:?})", source_device_path, verity_option); + + let cmd = "ls /dev/mapper"; + let output = Command::new("sh") + .arg("-c") + .arg(cmd) + .output() + .expect("KS (image-rs) Failed to execute 'ls' command"); + + if output.status.success() { + let stdout = str::from_utf8(&output.stdout) + .unwrap_or("KS Failed to decode stdout as UTF-8"); + + for line in stdout.split('\n') { + println!("KS mapper file: {}", line); + } + } else { + let stderr = str::from_utf8(&output.stderr) + .unwrap_or("KS Failed to decode stderr as UTF-8"); + eprintln!("KS Failed to execute '{}': {}", cmd, stderr); + } + let dm = DM::new()?; let verity_name = DmName::new(&verity_option.hash)?; let id = DevId::Name(verity_name); @@ -173,21 +259,58 @@ pub fn create_verity_device( verity_option.hash, "-", ); + //println!("CSG-M4GIC: (KS-image-rs) dm_verity params: ({:?})", verity_params); + + start_udev(); + // Mapping table in device mapper: : // is 0 // is size of device in sectors, and one sector is equal to 512 bytes. // is name of mapping target, here "verity" for dm-verity // are parameters for verity target + let verity_table = vec![( - 0, - verity_option.blocknum * verity_option.blocksize / 512, - "verity".into(), - verity_params, + 0, + verity_option.blocknum * verity_option.blocksize / 512, + "verity".into(), + verity_params, )]; - dm.device_create(verity_name, None, opts)?; - dm.table_load(&id, verity_table.as_slice(), opts)?; - dm.device_suspend(&id, opts)?; + //dm.device_create(verity_name, None, opts)?; + let dev = dm.device_create(verity_name, None, opts).unwrap(); + + println!("CSG-M4GIC: (KS-image-rs) Device created: {:?}", dev); + + // if let Err(ref e) = result { + // println!("CSG-M4GIC: (KS-image-rs) Error occurred while creating device: {}", e); + // result.unwrap(); + // } + + let dev_info = dm.table_load(&id, verity_table.as_slice(), opts)?; + + //println!("CSG-M4GIC: KS (image-rs) Loaded table with dev info: {:?}", dev_info); + + //println!("KS (image-rs) verity table loaded"); + + //println!("KS (image-rs) dev info collected: {:?}", dev_info); + + let devs = dm.list_devices(); + + //println!("CSG-M4GIC: KS (image-rs) listing devices: {:?}", devs); + + //dm.device_suspend(&id, opts)?; + + let result = dm.device_suspend(&id, opts); + println!("KS (image-rs) Device suspended result {:?}", result); + match result { + Ok(device_info) => { + println!("KS (image-rs) Device suspended successfully. Device info: {:?}", device_info); + }, + Err(e) => { + println!("KS (image-rs) Error occurred while trying to suspend device: {:?}", e); + } + } + println!("CSG-M4GIC: END: (KS-image-rs) create_verity_device"); Ok(format!("/dev/mapper/{}", &verity_option.hash)) }