From 203da1eb666fd164b5438b29782453d3cba7b5df Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 13 Nov 2024 17:09:21 -0800 Subject: [PATCH 01/17] build: drop non-rust support temporarily --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index b3a7dcfe..f111bfc4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1024,7 +1024,8 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .short('l') .long("language") .help("Programming language of the template") - .value_parser(["rust", "python", "javascript"]) + .value_parser(["rust"]) + //.value_parser(["rust", "python", "javascript"]) // TODO: resupport .default_value("rust") ) .arg(Arg::new("TEMPLATE") From c9a1fbac7a40739a41d6492d1bfed4e46f6e1daa Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 13 Nov 2024 21:38:45 -0800 Subject: [PATCH 02/17] new: get file-transfer test working --- src/new/mod.rs | 5 + .../file-transfer/file-transfer/Cargo.toml | 3 + .../file-transfer/file-transfer/src/lib.rs | 58 ++++- .../api/file-transfer-test:template.os-v0.wit | 1 + .../file-transfer-test/Cargo.toml | 3 - .../file-transfer-test/src/lib.rs | 211 +++++++++++++----- .../rust/no-ui/file-transfer/test/tests.toml | 1 + 7 files changed, 217 insertions(+), 65 deletions(-) diff --git a/src/new/mod.rs b/src/new/mod.rs index 07eda337..38290a56 100644 --- a/src/new/mod.rs +++ b/src/new/mod.rs @@ -171,6 +171,11 @@ fn replace_vars( .replace( &format!("{template_package_name}-"), &format!("{package_name_kebab}-"), + ) + // function call + .replace( + &format!("{template_package_name}("), + &format!("{package_name_snake}("), ); let input = if extension == "wit" { input diff --git a/src/new/templates/rust/no-ui/file-transfer/file-transfer/Cargo.toml b/src/new/templates/rust/no-ui/file-transfer/file-transfer/Cargo.toml index 25514d57..c9ffaa83 100644 --- a/src/new/templates/rust/no-ui/file-transfer/file-transfer/Cargo.toml +++ b/src/new/templates/rust/no-ui/file-transfer/file-transfer/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition = "2021" publish = false +[features] +test = [] + [dependencies] anyhow = "1.0" kinode_process_lib = { version = "0.9.6", features = ["logging"] } diff --git a/src/new/templates/rust/no-ui/file-transfer/file-transfer/src/lib.rs b/src/new/templates/rust/no-ui/file-transfer/file-transfer/src/lib.rs index 775ecbed..999b6491 100644 --- a/src/new/templates/rust/no-ui/file-transfer/file-transfer/src/lib.rs +++ b/src/new/templates/rust/no-ui/file-transfer/file-transfer/src/lib.rs @@ -8,8 +8,8 @@ use crate::kinode::process::file_transfer_worker::{ use crate::kinode::process::standard::{Address as WitAddress, ProcessId as WitProcessId}; use kinode_process_lib::logging::{error, info, init_logging, Level}; use kinode_process_lib::{ - await_message, call_init, println, - vfs::{create_drive, metadata, open_dir, Directory, FileType}, + await_message, call_init, get_capability, println, + vfs::{create_drive, metadata, open_dir, open_file, Directory, FileType}, Address, Message, ProcessId, Response, }; @@ -157,6 +157,57 @@ fn handle_message(our: &Address, message: &Message, files_dir: &Directory) -> an } } +#[cfg(feature = "test")] +#[derive(Debug, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto)] +#[cfg(feature = "test")] +enum Setup { + Caps, + WriteFile { name: String, contents: Vec }, +} + +#[cfg(feature = "test")] +fn handle_tester_setup(our: &Address, drive_path: &str) -> anyhow::Result<()> { + println!("awaiting setup..."); + + let Ok(message) = await_message() else { + return Err(anyhow::anyhow!("a")); + }; + // TODO: confirm its from tester + match message.body().try_into()? { + Setup::Caps => { + println!("got caps..."); + let vfs_read_cap = serde_json::json!({ + "kind": "read", + "drive": drive_path, + }) + .to_string(); + let vfs_address = Address { + node: our.node.clone(), + process: "vfs:distro:sys".parse()?, + }; + + let read_cap = get_capability(&vfs_address, &vfs_read_cap).unwrap(); + + Response::new() + .body(vec![]) + .capabilities(vec![read_cap]) + .send() + .unwrap(); + println!("sent caps"); + } + Setup::WriteFile { + ref name, + ref contents, + } => { + println!("got write file..."); + let file = open_file(&format!("{drive_path}/{name}"), true, None)?; + file.write(contents)?; + } + } + println!("setup done"); + Ok(()) +} + call_init!(init); fn init(our: Address) { init_logging(&our, Level::DEBUG, Level::INFO, None, None).unwrap(); @@ -165,6 +216,9 @@ fn init(our: Address) { let drive_path = create_drive(our.package_id(), "files", None).unwrap(); let files_dir = open_dir(&drive_path, false, None).unwrap(); + #[cfg(feature = "test")] + handle_tester_setup(&our, &drive_path).unwrap(); + loop { match await_message() { Err(send_error) => error!("got SendError: {send_error}"), diff --git a/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/api/file-transfer-test:template.os-v0.wit b/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/api/file-transfer-test:template.os-v0.wit index 99b62ce1..7c61a641 100644 --- a/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/api/file-transfer-test:template.os-v0.wit +++ b/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/api/file-transfer-test:template.os-v0.wit @@ -1,5 +1,6 @@ world file-transfer-test-template-dot-os-v0 { import file-transfer; + import file-transfer-worker; import tester; include process-v0; } diff --git a/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/Cargo.toml b/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/Cargo.toml index 38908c67..6bf0ef37 100644 --- a/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/Cargo.toml +++ b/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/Cargo.toml @@ -6,13 +6,10 @@ publish = false [dependencies] anyhow = "1.0" -bincode = "1.3" kinode_process_lib = "0.9.2" process_macros = { git = "https://github.com/kinode-dao/process_macros", rev = "626e501" } -rmp-serde = "1.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -thiserror = "1.0" wit-bindgen = "0.24.0" [lib] diff --git a/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/src/lib.rs b/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/src/lib.rs index 20fb005b..b59c5fa2 100644 --- a/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/src/lib.rs +++ b/src/new/templates/rust/no-ui/file-transfer/test/file-transfer-test/file-transfer-test/src/lib.rs @@ -1,6 +1,8 @@ use crate::kinode::process::file_transfer::{ - Address as WitAddress, DownloadRequest, FileInfo, InitializeRequest, ProgressRequest, - Request as TransferRequest, Response as TransferResponse, WorkerRequest, + FileInfo, Request as TransferRequest, Response as TransferResponse, +}; +use crate::kinode::process::file_transfer_worker::{ + Address as WitAddress, DownloadRequest, Request as WorkerRequest, }; use crate::kinode::process::standard::ProcessId as WitProcessId; use crate::kinode::process::tester::{ @@ -8,7 +10,8 @@ use crate::kinode::process::tester::{ }; use kinode_process_lib::{ - await_message, call_init, print_to_terminal, println, Address, ProcessId, Request, Response, + await_message, call_init, our_capabilities, print_to_terminal, println, save_capabilities, + vfs::File, Address, ProcessId, Request, Response, }; mod tester_lib; @@ -20,36 +23,137 @@ wit_bindgen::generate!({ additional_derives: [PartialEq, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); -fn test_list_files(address: &Address) -> anyhow::Result> { +const FILE_NAME: &str = "my_file.txt"; +const FILE_CONTENTS: &str = "hi"; +const DRIVE_PATH: &str = "file-transfer:template.os"; + +impl From
for WitAddress { + fn from(address: Address) -> Self { + WitAddress { + node: address.node, + process: address.process.into(), + } + } +} + +impl From for WitProcessId { + fn from(process: ProcessId) -> Self { + WitProcessId { + process_name: process.process_name, + package_name: process.package_name, + publisher_node: process.publisher_node, + } + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto)] +enum Setup { + Caps, + WriteFile { name: String, contents: Vec }, +} + +fn make_ft_address(node: &str) -> Address { + Address { + node: node.to_string(), + process: ProcessId::new(Some("file-transfer"), "file-transfer", "template.os"), + } +} + +fn make_file_path() -> String { + format!("{DRIVE_PATH}/files/{FILE_NAME}") +} + +fn setup(our: &Address, their: &str) -> anyhow::Result<()> { + let our_ft_address = make_ft_address(&our.node); + let their_ft_address = make_ft_address(their); + + // write file on their + Request::new() + .target(their_ft_address.clone()) + .body(Setup::WriteFile { + name: FILE_NAME.to_string(), + contents: FILE_CONTENTS.as_bytes().to_vec(), + }) + .send()?; + + // caps on our + println!("file-transfer-test: started caps handshake..."); + + let response = Request::new() + .target(our_ft_address.clone()) + .body(Setup::Caps) + .send_and_await_response(5)??; + + save_capabilities(response.capabilities()); + println!("file-transfer-test: got caps {:#?}", our_capabilities()); + + Ok(()) +} + +fn test_list_files(our_ft_address: &Address, their_ft_address: &Address) -> anyhow::Result<()> { + // our: none + let response = Request::new() + .target(our_ft_address) + .body(TransferRequest::ListFiles) + .send_and_await_response(15)? + .unwrap(); + if response.is_request() { + fail!("file-transfer-test"); + }; + let TransferResponse::ListFiles(files) = response.body().try_into()?; + println!("{files:?}"); + if files.len() != 0 { + fail!("file-transfer-test"); + } + + // their: one let response = Request::new() - .target(address) + .target(their_ft_address) .body(TransferRequest::ListFiles) .send_and_await_response(15)? .unwrap(); if response.is_request() { fail!("file-transfer-test"); }; - let TransferResponse::ListFiles(files) = response.body().try_into()? else { + let TransferResponse::ListFiles(files) = response.body().try_into()?; + println!("{files:?}"); + if files.len() != 1 { fail!("file-transfer-test"); + } + let file = files[0].clone(); + let expected_file_info = FileInfo { + name: make_file_path(), + size: FILE_CONTENTS.len() as u64, }; - Ok(files) + if file != expected_file_info { + fail!("file-transfer-test"); + } + Ok(()) } -fn test_download(name: String, our: &Address, address: &Address) -> anyhow::Result<()> { +fn test_download(our_ft_address: &Address, their_ft_address: &Address) -> anyhow::Result<()> { let response = Request::new() - .target(our) - .body(TransferRequest::Download(DownloadRequest { - name, - target: address, + .target(our_ft_address) + .body(WorkerRequest::Download(DownloadRequest { + name: FILE_NAME.to_string(), + target: their_ft_address.clone().into(), + is_requestor: true, })) .send_and_await_response(15)? .unwrap(); if response.is_request() { fail!("file-transfer-test"); }; - let TransferResponse::Download = response.body().try_into()? else { - fail!("file-transfer-test"); + std::thread::sleep(std::time::Duration::from_secs(3)); + + let file = File { + path: make_file_path(), + timeout: 5, }; + let file_contents = file.read()?; + if file_contents != FILE_CONTENTS.as_bytes() { + fail!("file-transfer-test"); + } Ok(()) } @@ -72,60 +176,47 @@ fn handle_message(our: &Address) -> anyhow::Result<()> { }) = message.body().try_into()?; print_to_terminal(0, "file-transfer-test: a"); assert!(node_names.len() >= 2); - if our.node != node_names[0] { - // we are not master node: return - Response::new() - .body(TesterResponse::Run(Ok(()))) - .send() - .unwrap(); - return Ok(()); - } - // we are master node + assert!(our.node == node_names[0]); - let our_ft_address = Address { - node: our.node.clone(), - process: ProcessId::new(Some("file-transfer"), "file-transfer", "template.os"), - }; - let their_ft_address = Address { - node: node_names[1].clone(), - process: ProcessId::new(Some("file-transfer"), "file-transfer", "template.os"), - }; + if setup(&our, &node_names[1]).is_err() { + fail!("file-transfer-test"); + } - // Send - print_to_terminal(0, "file-transfer-test: b"); - let message: String = "hello".into(); - let _ = Request::new() - .target(our_ft_address.clone()) - .body(ChatRequest::Send(SendRequest { - target: node_names[1].clone(), - message: message.clone(), - })) - .send_and_await_response(15)? - .unwrap(); + let our_ft_address = make_ft_address(&our.node); + let their_ft_address = make_ft_address(&node_names[1]); - // Get history from receiver & test - print_to_terminal(0, "file-transfer-test: c"); - let response = Request::new() - .target(their_ft_address.clone()) - .body(ChatRequest::History(our.node.clone())) - .send_and_await_response(15)? - .unwrap(); - if response.is_request() { + if test_list_files(&our_ft_address, &their_ft_address).is_err() { fail!("file-transfer-test"); - }; - let ChatResponse::History(messages) = response.body().try_into()? else { - fail!("file-transfer-test"); - }; - let expected_messages = vec![ChatMessage { - author: our.node.clone(), - content: message, - }]; + } - if messages != expected_messages { - println!("{messages:?} != {expected_messages:?}"); + // Test file_transfer_worker + println!("file-transfer-test: b"); + if test_download(&our_ft_address, &their_ft_address).is_err() { fail!("file-transfer-test"); } + //let response = Request::new() + // .target(our_ft_address.clone()) + // .body(WorkerRequest::Download(DownloadRequest { + // name: FILE_NAME.to_string(), + // target: their_ft_address.into(), + // is_requestor: true, + // })) + // .send_and_await_response(15)? + // .unwrap(); + //if response.is_request() { + // fail!("file-transfer-test"); + //}; + //std::thread::sleep(std::time::Duration::from_secs(3)); + + //let file = File { + // path: format!("{DRIVE_PATH}/files/{FILE_NAME}"), + // timeout: 5, + //}; + //let file_contents = file.read()?; + //if file_contents != FILE_CONTENTS.as_bytes() { + // fail!("file-transfer-test"); + //} Response::new() .body(TesterResponse::Run(Ok(()))) diff --git a/src/new/templates/rust/no-ui/file-transfer/test/tests.toml b/src/new/templates/rust/no-ui/file-transfer/test/tests.toml index 976110d0..a354839b 100644 --- a/src/new/templates/rust/no-ui/file-transfer/test/tests.toml +++ b/src/new/templates/rust/no-ui/file-transfer/test/tests.toml @@ -5,6 +5,7 @@ runtime_build_release = false [[tests]] +dependency_package_paths = [".."] setup_packages = [ { path = "..", run = true } ] From b0c6807e2f32a64e9ed6006d35fd2fffbdc61cd2 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 14 Nov 2024 15:20:31 -0800 Subject: [PATCH 03/17] new: update chat with UI --- build.rs | 4 +- .../templates/rust/no-ui/chat/chat/src/lib.rs | 48 ++-- .../rust/ui/chat/api/chat:template.os-v0.wit | 27 ++ .../templates/rust/ui/chat/chat/Cargo.toml | 4 +- .../templates/rust/ui/chat/chat/src/lib.rs | 266 ++++++++---------- 5 files changed, 170 insertions(+), 179 deletions(-) create mode 100644 src/new/templates/rust/ui/chat/api/chat:template.os-v0.wit diff --git a/build.rs b/build.rs index 8388c6bb..8bfdc6e8 100644 --- a/build.rs +++ b/build.rs @@ -69,7 +69,7 @@ fn visit_dirs(dir: &Path, output_buffer: &mut Vec) -> io::Result<()> { let relative_path = path.strip_prefix(TEMPLATES_DIR).unwrap(); let path_str = relative_path.to_str().unwrap().replace("\\", "/"); - let relative_path_from_includes = Path::new("..").join(path); + let relative_path_from_includes = Path::new("..").join(&path); let path_str_from_includes = relative_path_from_includes .to_str() .unwrap() @@ -79,6 +79,7 @@ fn visit_dirs(dir: &Path, output_buffer: &mut Vec) -> io::Result<()> { " (\"{}\", include_str!(\"{}\")),", path_str, path_str_from_includes, )?; + println!("cargo::rerun-if-changed={}", path.display()); } } Ok(()) @@ -104,6 +105,7 @@ fn make_chain_includes() -> anyhow::Result<()> { commit, Path::new("..").join(&path).display(), )?; + println!("cargo::rerun-if-changed={}", path.display()); } writeln!(&mut output_buffer, "];")?; diff --git a/src/new/templates/rust/no-ui/chat/chat/src/lib.rs b/src/new/templates/rust/no-ui/chat/chat/src/lib.rs index 09a75c1e..84c69420 100644 --- a/src/new/templates/rust/no-ui/chat/chat/src/lib.rs +++ b/src/new/templates/rust/no-ui/chat/chat/src/lib.rs @@ -31,34 +31,33 @@ fn handle_message( ref target, ref message, }) => { + // Counterparty is the other node in the chat with us + let (counterparty, author) = if target == &our.node { + (&source.node, source.node.clone()) + } else { + (target, our.node.clone()) + }; + + // If the target is not us, send a request to the target if target == &our.node { println!("{}: {}", source.node, message); - let message = ChatMessage { - author: source.node.clone(), - content: message.into(), - }; - message_archive - .entry(source.node.clone()) - .and_modify(|e| e.push(message.clone())) - .or_insert(vec![message]); } else { - let _ = Request::new() - .target(Address { - node: target.clone(), - process: "chat:chat:template.os".parse()?, - }) + Request::new() + .target((target, "chat", "chat", "template.os")) .body(body) - .send_and_await_response(5)? - .unwrap(); - let message = ChatMessage { - author: our.node.clone(), - content: message.into(), - }; - message_archive - .entry(target.clone()) - .and_modify(|e| e.push(message.clone())) - .or_insert(vec![message]); + .send_and_await_response(5)??; } + + // Insert message into archive, creating one for counterparty if it DNE + let new_message = ChatMessage { + author: author.clone(), + content: message.clone(), + }; + message_archive + .entry(counterparty.to_string()) + .and_modify(|e| e.push(new_message.clone())) + .or_insert(vec![new_message]); + Response::new().body(ChatResponse::Send).send().unwrap(); } ChatRequest::History(ref node) => { @@ -69,8 +68,7 @@ fn handle_message( .map(|msgs| msgs.clone()) .unwrap_or_default(), )) - .send() - .unwrap(); + .send()?; } } Ok(()) diff --git a/src/new/templates/rust/ui/chat/api/chat:template.os-v0.wit b/src/new/templates/rust/ui/chat/api/chat:template.os-v0.wit new file mode 100644 index 00000000..d883229b --- /dev/null +++ b/src/new/templates/rust/ui/chat/api/chat:template.os-v0.wit @@ -0,0 +1,27 @@ +interface chat { + variant request { + send(send-request), + /// history of chat with given node + history(string), + } + + variant response { + send, + history(list), + } + + record send-request { + target: string, + message: string, + } + + record chat-message { + author: string, + content: string, + } +} + +world chat-template-dot-os-v0 { + import chat; + include process-v0; +} diff --git a/src/new/templates/rust/ui/chat/chat/Cargo.toml b/src/new/templates/rust/ui/chat/chat/Cargo.toml index 42cb0fee..e1a487d4 100644 --- a/src/new/templates/rust/ui/chat/chat/Cargo.toml +++ b/src/new/templates/rust/ui/chat/chat/Cargo.toml @@ -5,8 +5,8 @@ edition = "2021" [dependencies] anyhow = "1.0" -bincode = "1.3.3" -kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib", tag = "v0.8.0" } +kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib", rev = "51369db", features = ["logging"] } +process_macros = { git = "https://github.com/kinode-dao/process_macros", rev = "626e501" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" wit-bindgen = "0.24.0" diff --git a/src/new/templates/rust/ui/chat/chat/src/lib.rs b/src/new/templates/rust/ui/chat/chat/src/lib.rs index 687b0520..0680cbc4 100644 --- a/src/new/templates/rust/ui/chat/chat/src/lib.rs +++ b/src/new/templates/rust/ui/chat/chat/src/lib.rs @@ -1,40 +1,29 @@ use std::collections::HashMap; -use std::str::FromStr; +use crate::kinode::process::chat::{ + ChatMessage, Request as ChatRequest, Response as ChatResponse, SendRequest, +}; +use kinode_process_lib::logging::{error, info, init_logging, Level}; use kinode_process_lib::{ await_message, call_init, get_blob, - http::{ - bind_http_path, bind_ws_path, send_response, send_ws_push, serve_ui, HttpServerRequest, - StatusCode, WsMessageType, + http::server::{ + send_response, HttpBindingConfig, HttpServer, HttpServerRequest, StatusCode, + WsBindingConfig, WsMessageType, }, - println, Address, LazyLoadBlob, Message, ProcessId, Request, Response, + println, Address, LazyLoadBlob, Message, Request, Response, }; -use serde::{Deserialize, Serialize}; wit_bindgen::generate!({ path: "target/wit", - world: "process-v0", + world: "chat-template-dot-os-v0", + generate_unused_types: true, + additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); -#[derive(Debug, Serialize, Deserialize)] -enum ChatRequest { - Send { target: String, message: String }, - History, -} - -#[derive(Debug, Serialize, Deserialize)] -enum ChatResponse { - Ack, - History { messages: MessageArchive }, -} +const HTTP_API_PATH: &str = "/messages"; +const WS_PATH: &str = "/"; -#[derive(Debug, Serialize, Deserialize, Clone)] -struct ChatMessage { - author: String, - content: String, -} - -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize, process_macros::SerdeJsonInto)] struct NewMessage { chat: String, author: String, @@ -43,24 +32,28 @@ struct NewMessage { type MessageArchive = HashMap>; +fn make_http_address(our: &Address) -> Address { + Address::from((our.node(), "http_server", "distro", "sys")) +} + fn handle_http_server_request( our: &Address, - message_archive: &mut MessageArchive, - our_channel_id: &mut u32, - source: &Address, body: &[u8], + message_archive: &mut MessageArchive, + server: &mut HttpServer, ) -> anyhow::Result<()> { - let Ok(server_request) = serde_json::from_slice::(body) else { - // Fail silently if we can't parse the request + let Ok(request) = serde_json::from_slice::(body) else { + // Fail quietly if we can't parse the request + info!("couldn't parse message from http_server: {body:?}"); return Ok(()); }; - match server_request { - HttpServerRequest::WebSocketOpen { channel_id, .. } => { - // Set our channel_id to the newly opened channel - // Note: this code could be improved to support multiple channels - *our_channel_id = channel_id; - } + match request { + HttpServerRequest::WebSocketOpen { + ref path, + channel_id, + } => server.handle_websocket_open(path, channel_id), + HttpServerRequest::WebSocketClose(channel_id) => server.handle_websocket_close(channel_id), HttpServerRequest::WebSocketPush { .. } => { let Some(blob) = get_blob() else { return Ok(()); @@ -68,51 +61,52 @@ fn handle_http_server_request( handle_chat_request( our, - message_archive, - our_channel_id, - source, + &make_http_address(our), &blob.bytes, - false, + true, + message_archive, + server, )?; } - HttpServerRequest::WebSocketClose(_channel_id) => {} HttpServerRequest::Http(request) => { - match request.method()?.as_str() { + match request.method().unwrap().as_str() { // Get all messages "GET" => { - let mut headers = HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); + let headers = HashMap::from([( + "Content-Type".to_string(), + "application/json".to_string(), + )]); send_response( StatusCode::OK, Some(headers), - serde_json::to_vec(&ChatResponse::History { - messages: message_archive.clone(), - }) + serde_json::to_vec(&serde_json::json!({ + "History": { + "messages": message_archive.clone() + } + })) .unwrap(), ); } // Send a message "POST" => { let Some(blob) = get_blob() else { + send_response(StatusCode::BAD_REQUEST, None, vec![]); return Ok(()); }; handle_chat_request( our, - message_archive, - our_channel_id, - source, + &make_http_address(our), &blob.bytes, true, - )?; + message_archive, + server, + ) + .unwrap(); - // Send an http response via the http server send_response(StatusCode::CREATED, None, vec![]); } - _ => { - // Method not allowed - send_response(StatusCode::METHOD_NOT_ALLOWED, None, vec![]); - } + _ => send_response(StatusCode::METHOD_NOT_ALLOWED, None, vec![]), } } }; @@ -122,23 +116,18 @@ fn handle_http_server_request( fn handle_chat_request( our: &Address, - message_archive: &mut MessageArchive, - channel_id: &mut u32, source: &Address, body: &[u8], is_http: bool, + message_archive: &mut MessageArchive, + server: &HttpServer, ) -> anyhow::Result<()> { - let Ok(chat_request) = serde_json::from_slice::(body) else { - // Fail silently if we can't parse the request - return Ok(()); - }; - - match chat_request { - ChatRequest::Send { + match body.try_into()? { + ChatRequest::Send(SendRequest { ref target, ref message, - } => { - // counterparty will be the other node in the chat with us + }) => { + // Counterparty is the other node in the chat with us let (counterparty, author) = if target == &our.node { (&source.node, source.node.clone()) } else { @@ -150,106 +139,74 @@ fn handle_chat_request( println!("{}: {}", source.node, message); } else { Request::new() - .target(Address { - node: target.clone(), - process: ProcessId::from_str("chat:chat:template.os")?, - }) + .target((target, "chat", "chat", "template.os")) .body(body) .send_and_await_response(5)??; } - // Retreive the message archive for the counterparty, or create a new one if it doesn't exist - let messages = match message_archive.get_mut(counterparty) { - Some(messages) => messages, - None => { - message_archive.insert(counterparty.clone(), Vec::new()); - message_archive.get_mut(counterparty).unwrap() - } - }; - + // Insert message into archive, creating one for counterparty if it DNE let new_message = ChatMessage { author: author.clone(), content: message.clone(), }; + message_archive + .entry(counterparty.to_string()) + .and_modify(|e| e.push(new_message.clone())) + .or_insert(vec![new_message]); - // If this is an HTTP request, handle the response in the calling function if is_http { - // Add the new message to the archive - messages.push(new_message); + // If is HTTP from FE: done return Ok(()); } - // If this is not an HTTP request, send a response to the other node - Response::new() - .body(serde_json::to_vec(&ChatResponse::Ack).unwrap()) - .send() - .unwrap(); + // Not HTTP from FE: send response to node & update any FE listeners + Response::new().body(ChatResponse::Send).send().unwrap(); - // Add the new message to the archive - messages.push(new_message); - - // Generate a blob for the new message + // Send a WebSocket message to the http server in order to update the UI let blob = LazyLoadBlob { mime: Some("application/json".to_string()), - bytes: serde_json::json!({ + bytes: serde_json::to_vec(&serde_json::json!({ "NewMessage": NewMessage { - chat: counterparty.clone(), + chat: counterparty.to_string(), author, - content: message.clone(), + content: message.to_string(), } - }) - .to_string() - .as_bytes() - .to_vec(), + })) + .unwrap(), }; - - // Send a WebSocket message to the http server in order to update the UI - send_ws_push( - channel_id.clone(), - WsMessageType::Text, - blob, - ); + server.ws_push_all_channels(WS_PATH, WsMessageType::Text, blob); } - ChatRequest::History => { - // If this is an HTTP request, send a response to the http server - + ChatRequest::History(ref node) => { Response::new() - .body( - serde_json::to_vec(&ChatResponse::History { - messages: message_archive.clone(), - }) - .unwrap(), - ) - .send() - .unwrap(); + .body(ChatResponse::History( + message_archive + .get(node) + .map(|msgs| msgs.clone()) + .unwrap_or_default(), + )) + .send()?; } - }; - + } Ok(()) } fn handle_message( our: &Address, + message: &Message, message_archive: &mut MessageArchive, - channel_id: &mut u32, + server: &mut HttpServer, ) -> anyhow::Result<()> { - let message = await_message().unwrap(); + if !message.is_request() { + return Ok(()); + } - match message { - Message::Response { .. } => { - println!("got response - {:?}", message); - return Ok(()); - } - Message::Request { - ref source, - ref body, - .. - } => { - // Requests that come from other nodes running this app - handle_chat_request(our, message_archive, channel_id, source, body, false)?; - // Requests that come from our http server - handle_http_server_request(our, message_archive, channel_id, source, body)?; - } + let body = message.body(); + let source = message.source(); + + if source == &make_http_address(our) { + handle_http_server_request(our, body, message_archive, server)?; + } else { + handle_chat_request(our, source, body, false, message_archive, server)?; } Ok(()) @@ -257,26 +214,33 @@ fn handle_message( call_init!(init); fn init(our: Address) { - println!("begin"); - - let mut message_archive: MessageArchive = HashMap::new(); - let mut channel_id = 0; + init_logging(&our, Level::DEBUG, Level::INFO, None, None).unwrap(); + info!("begin"); - // Bind UI files to routes; index.html is bound to "/" - serve_ui(&our, "ui", true, true, vec!["/"]).unwrap(); + let mut message_archive = HashMap::new(); - // Bind HTTP path /messages - bind_http_path("/messages", true, false).unwrap(); + let mut server = HttpServer::new(5); - // Bind WebSocket path - bind_ws_path("/", true, false).unwrap(); + // Bind UI files to routes with index.html at "/"; API to /messages; WS to "/" + server + .serve_ui(&our, "ui", vec!["/"], HttpBindingConfig::default()) + .expect("failed to serve UI"); + server + .bind_http_path(HTTP_API_PATH, HttpBindingConfig::default()) + .expect("failed to bind messages API"); + server + .bind_ws_path(WS_PATH, WsBindingConfig::default()) + .expect("failed to bind WS API"); loop { - match handle_message(&our, &mut message_archive, &mut channel_id) { - Ok(()) => {} - Err(e) => { - println!("error: {:?}", e); + match await_message() { + Err(send_error) => error!("got SendError: {send_error}"), + Ok(ref message) => { + match handle_message(&our, message, &mut message_archive, &mut server) { + Ok(_) => {} + Err(e) => error!("got error while handling message: {e:?}"), + } } - }; + } } } From d772b04e7826d3f2e7fdc2423fb0f9145fe38b01 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 14 Nov 2024 15:23:29 -0800 Subject: [PATCH 04/17] new: add no-ui chat test to ui chat --- .../rust/ui/chat/test/chat-test/Cargo.toml | 10 ++ .../api/chat_test:template.os-v0.wit | 5 + .../chat/test/chat-test/chat-test/Cargo.toml | 22 ++++ .../chat/test/chat-test/chat-test/src/lib.rs | 107 ++++++++++++++++++ .../chat-test/chat-test/src/tester_lib.rs | 30 +++++ .../rust/ui/chat/test/chat-test/metadata.json | 21 ++++ .../ui/chat/test/chat-test/pkg/manifest.json | 15 +++ .../templates/rust/ui/chat/test/tests.toml | 28 +++++ 8 files changed, 238 insertions(+) create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/Cargo.toml create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/api/chat_test:template.os-v0.wit create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/chat-test/Cargo.toml create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/lib.rs create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/tester_lib.rs create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/metadata.json create mode 100644 src/new/templates/rust/ui/chat/test/chat-test/pkg/manifest.json create mode 100644 src/new/templates/rust/ui/chat/test/tests.toml diff --git a/src/new/templates/rust/ui/chat/test/chat-test/Cargo.toml b/src/new/templates/rust/ui/chat/test/chat-test/Cargo.toml new file mode 100644 index 00000000..1a52ab80 --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/Cargo.toml @@ -0,0 +1,10 @@ +[workspace] +resolver = "2" +members = [ + "chat-test", +] + +[profile.release] +panic = "abort" +opt-level = "s" +lto = true diff --git a/src/new/templates/rust/ui/chat/test/chat-test/api/chat_test:template.os-v0.wit b/src/new/templates/rust/ui/chat/test/chat-test/api/chat_test:template.os-v0.wit new file mode 100644 index 00000000..c87f01eb --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/api/chat_test:template.os-v0.wit @@ -0,0 +1,5 @@ +world chat-test-template-dot-os-v0 { + import chat; + import tester; + include process-v0; +} diff --git a/src/new/templates/rust/ui/chat/test/chat-test/chat-test/Cargo.toml b/src/new/templates/rust/ui/chat/test/chat-test/chat-test/Cargo.toml new file mode 100644 index 00000000..1c280c2c --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/chat-test/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "chat-test" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1.0" +bincode = "1.3" +kinode_process_lib = "0.9.2" +process_macros = { git = "https://github.com/kinode-dao/process_macros", rev = "626e501" } +rmp-serde = "1.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "1.0" +wit-bindgen = "0.24.0" + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "kinode:process" diff --git a/src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/lib.rs b/src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/lib.rs new file mode 100644 index 00000000..40e55a74 --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/lib.rs @@ -0,0 +1,107 @@ +use crate::kinode::process::chat::{ChatMessage, Request as ChatRequest, Response as ChatResponse, SendRequest}; +use crate::kinode::process::tester::{Request as TesterRequest, Response as TesterResponse, RunRequest, FailResponse}; + +use kinode_process_lib::{await_message, call_init, print_to_terminal, println, Address, ProcessId, Request, Response}; + +mod tester_lib; + +wit_bindgen::generate!({ + path: "target/wit", + world: "chat-test-template-dot-os-v0", + generate_unused_types: true, + additional_derives: [PartialEq, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], +}); + +fn handle_message (our: &Address) -> anyhow::Result<()> { + let message = await_message().unwrap(); + + if !message.is_request() { + unimplemented!(); + } + let source = message.source(); + if our.node != source.node { + return Err(anyhow::anyhow!( + "rejecting foreign Message from {:?}", + source, + )); + } + let TesterRequest::Run(RunRequest { + input_node_names: node_names, + .. + }) = message.body().try_into()?; + print_to_terminal(0, "chat_test: a"); + assert!(node_names.len() >= 2); + if our.node != node_names[0] { + // we are not master node: return + Response::new() + .body(TesterResponse::Run(Ok(()))) + .send() + .unwrap(); + return Ok(()); + } + + // we are master node + + let our_chat_address = Address { + node: our.node.clone(), + process: ProcessId::new(Some("chat"), "chat", "template.os"), + }; + let their_chat_address = Address { + node: node_names[1].clone(), + process: ProcessId::new(Some("chat"), "chat", "template.os"), + }; + + // Send + print_to_terminal(0, "chat_test: b"); + let message: String = "hello".into(); + let _ = Request::new() + .target(our_chat_address.clone()) + .body(ChatRequest::Send(SendRequest { + target: node_names[1].clone(), + message: message.clone(), + })) + .send_and_await_response(15)?.unwrap(); + + // Get history from receiver & test + print_to_terminal(0, "chat_test: c"); + let response = Request::new() + .target(their_chat_address.clone()) + .body(ChatRequest::History(our.node.clone())) + .send_and_await_response(15)?.unwrap(); + if response.is_request() { fail!("chat_test"); }; + let ChatResponse::History(messages) = response.body().try_into()? else { + fail!("chat_test"); + }; + let expected_messages = vec![ChatMessage { + author: our.node.clone(), + content: message, + }]; + + if messages != expected_messages { + println!("{messages:?} != {expected_messages:?}"); + fail!("chat_test"); + } + + Response::new() + .body(TesterResponse::Run(Ok(()))) + .send() + .unwrap(); + + Ok(()) +} + +call_init!(init); +fn init(our: Address) { + print_to_terminal(0, "begin"); + + loop { + match handle_message(&our) { + Ok(()) => {}, + Err(e) => { + print_to_terminal(0, format!("chat_test: error: {e:?}").as_str()); + + fail!("chat_test"); + }, + }; + } +} diff --git a/src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/tester_lib.rs b/src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/tester_lib.rs new file mode 100644 index 00000000..8fb7ae19 --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/chat-test/src/tester_lib.rs @@ -0,0 +1,30 @@ +#[allow(unused_imports)] +use crate::kinode::process::tester::{FailResponse, Response as TesterResponse}; + +#[macro_export] +macro_rules! fail { + ($test:expr) => { + Response::new() + .body(TesterResponse::Run(Err(FailResponse { + test: $test.into(), + file: file!().into(), + line: line!(), + column: column!(), + }))) + .send() + .unwrap(); + panic!("") + }; + ($test:expr, $file:expr, $line:expr, $column:expr) => { + Response::new() + .body(TesterResponse::Run(Err(FailResponse { + test: $test.into(), + file: $file.into(), + line: $line, + column: $column, + }))) + .send() + .unwrap(); + panic!("") + }; +} diff --git a/src/new/templates/rust/ui/chat/test/chat-test/metadata.json b/src/new/templates/rust/ui/chat/test/chat-test/metadata.json new file mode 100644 index 00000000..0d0d5a63 --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/metadata.json @@ -0,0 +1,21 @@ +{ + "name": "chat Test", + "description": "A test for chat.", + "image": "", + "properties": { + "package_name": "chat-test", + "current_version": "0.1.0", + "publisher": "template.os", + "mirrors": [], + "code_hashes": { + "0.1.0": "" + }, + "wit_version": 0, + "dependencies": [ + "chat:template.os", + "tester:sys" + ] + }, + "external_url": "", + "animation_url": "" +} diff --git a/src/new/templates/rust/ui/chat/test/chat-test/pkg/manifest.json b/src/new/templates/rust/ui/chat/test/chat-test/pkg/manifest.json new file mode 100644 index 00000000..b37a7f82 --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/chat-test/pkg/manifest.json @@ -0,0 +1,15 @@ +[ + { + "process_name": "chat-test", + "process_wasm_path": "/chat-test.wasm", + "on_exit": "Restart", + "request_networking": false, + "request_capabilities": [ + "chat:chat:template.os" + ], + "grant_capabilities": [ + "chat:chat:template.os" + ], + "public": true + } +] diff --git a/src/new/templates/rust/ui/chat/test/tests.toml b/src/new/templates/rust/ui/chat/test/tests.toml new file mode 100644 index 00000000..a15673e4 --- /dev/null +++ b/src/new/templates/rust/ui/chat/test/tests.toml @@ -0,0 +1,28 @@ +runtime = { FetchVersion = "latest" } +# runtime = { RepoPath = "~/git/kinode" } +persist_home = false +runtime_build_release = false + + +[[tests]] +dependency_package_paths = [".."] +setup_packages = [ + { path = "..", run = true } +] +setup_scripts = [] +test_package_paths = ["chat-test"] +test_scripts = [] +timeout_secs = 5 +fakechain_router = 8545 + +[[tests.nodes]] +port = 8080 +home = "home/first" +fake_node_name = "first.dev" +runtime_verbosity = 2 + +[[tests.nodes]] +port = 8081 +home = "home/second" +fake_node_name = "second.dev" +runtime_verbosity = 2 From 3252df7f7e0d6a4634ae4b426a24a06fa8a6ee13 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 14 Nov 2024 17:10:07 -0800 Subject: [PATCH 05/17] new: hotfix template find-and-replace --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/new/mod.rs | 142 ++++++++++++++++++++++++++++++------------------- 3 files changed, 89 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8211a7cf..ae6ac408 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,7 +2151,7 @@ dependencies = [ [[package]] name = "kit" -version = "0.8.3" +version = "0.8.4" dependencies = [ "alloy", "alloy-sol-macro", diff --git a/Cargo.toml b/Cargo.toml index 61f57576..f15de3b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kit" -version = "0.8.3" +version = "0.8.4" edition = "2021" [build-dependencies] diff --git a/src/new/mod.rs b/src/new/mod.rs index 38290a56..681cbf41 100644 --- a/src/new/mod.rs +++ b/src/new/mod.rs @@ -132,74 +132,106 @@ fn replace_vars( .map(|e| e.to_string()) .collect(); - let input = input + let replacements = vec![ // wit - .replace( - &format!("{template_package_name_kebab}-"), - &format!("{package_name_kebab}-"), - ) + ( + format!("{template_package_name_kebab}-"), + format!("{package_name_kebab}-"), + ), // rust imports - .replace( - &format!("{template_package_name_snake}::"), - &format!("{package_name_snake}::"), - ) + ( + format!("{template_package_name_snake}::"), + format!("{package_name_snake}::"), + ), // manifest.json - .replace( - &format!("{template_package_name_kebab}.wasm"), - &format!("{package_name_kebab}.wasm"), - ) + ( + format!("{template_package_name_kebab}.wasm"), + format!("{package_name_kebab}.wasm"), + ), // tests manifest.json - .replace( - &format!("{template_package_name_kebab}-test.wasm"), - &format!("{package_name_kebab}-test.wasm"), - ) + ( + format!("{template_package_name_kebab}-test.wasm"), + format!("{package_name_kebab}-test.wasm"), + ), // part of a var name - .replace( - &format!("{template_package_name}_"), - &format!("{package_name_snake}_"), - ) + ( + format!("{template_package_name}_"), + format!("{package_name_snake}_"), + ), // part of a var name - .replace( - &format!("_{template_package_name}"), - &format!("_{package_name_snake}"), - ) + ( + format!("_{template_package_name}"), + format!("_{package_name_snake}"), + ), // field in a struct - .replace( - &format!("{template_package_name}: "), - &format!("{package_name_snake}: "), - ) - .replace( - &format!("{template_package_name}-"), - &format!("{package_name_kebab}-"), - ) + ( + format!("{template_package_name}: "), + format!("{package_name_snake}: "), + ), + ( + format!("{template_package_name}-"), + format!("{package_name_kebab}-"), + ), // function call - .replace( - &format!("{template_package_name}("), - &format!("{package_name_snake}("), - ); - let input = if extension == "wit" { - input - .replace(&template_package_name_kebab, &package_name_kebab) - .replace(template_package_name, package_name) + ( + format!("{template_package_name}("), + format!("{package_name_snake}("), + ), + ]; + let mut replacements: Vec<(&str, &str)> = replacements + .iter() + .map(|(s, t)| (s.as_str(), t.as_str())) + .collect(); + if extension == "wit" { + replacements.append(&mut vec![ + (&template_package_name_kebab, &package_name_kebab), + (template_package_name, package_name), + ]); } else if js.contains(extension) { - input - .replace(template_package_name, &package_name_snake) - .replace(&template_package_name_kebab, &package_name_kebab) + replacements.append(&mut vec![ + (template_package_name, &package_name_snake), + (&template_package_name_kebab, &package_name_kebab), + ]); } else { - input - .replace(template_package_name, package_name) - .replace(&template_package_name_kebab, &package_name_kebab) - .replace(&template_package_name_snake, &package_name_snake) + replacements.append(&mut vec![ + (template_package_name, package_name), + (&template_package_name_kebab, &package_name_kebab), + (&template_package_name_snake, &package_name_snake), + ]); }; - input - .replace( + replacements.append(&mut vec![ + ( &template_package_name_upper_camel, &package_name_upper_camel, - ) - .replace("template.os", publisher) - .replace("template_dot_os", &publisher_dotted_snake) - .replace("template-dot-os", &publisher_dotted_kebab) - .replace("TemplateDotOs", &publisher_dotted_upper_camel) + ), + ("template.os", publisher), + ("template_dot_os", &publisher_dotted_snake), + ("template-dot-os", &publisher_dotted_kebab), + ("TemplateDotOs", &publisher_dotted_upper_camel), + ]); + + let pattern = replacements + .iter() + .map(|(from, _)| regex::escape(from)) + .collect::>() + .join("|"); + + let regex = regex::Regex::new(&pattern).unwrap(); + + regex + .replace_all(&input, |caps: ®ex::Captures| { + let matched = caps.get(0).unwrap().as_str().to_string(); + replacements + .iter() + .find_map(|(from, to)| { + if *from == matched.as_str() { + Some(to.to_string()) + } else { + None + } + }) + .unwrap_or(matched) + }) .to_string() } From 79d96655b094d2ca03519dcf5c749f0cd0555eaf Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 15 Nov 2024 11:13:42 -0800 Subject: [PATCH 06/17] bump version to 0.8.5 & update README --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae6ac408..e6770821 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,7 +2151,7 @@ dependencies = [ [[package]] name = "kit" -version = "0.8.4" +version = "0.8.5" dependencies = [ "alloy", "alloy-sol-macro", diff --git a/Cargo.toml b/Cargo.toml index f15de3b6..cad4c00f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kit" -version = "0.8.4" +version = "0.8.5" edition = "2021" [build-dependencies] diff --git a/README.md b/README.md index dd99e133..cfa9446f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Tool**kit** for developing on [Kinode OS](https://github.com/kinode-dao/kinode). -Documentation in the [Kinode Book](https://book.kinode.org/kit-dev-toolkit.html); example usage [here](https://book.kinode.org/my_first_app/chapter_1.html). +Documentation in the [Kinode Book](https://book.kinode.org/kit/kit-dev-toolkit.html); example usage [here](https://book.kinode.org/my_first_app/chapter_1.html). ## Installing @@ -71,7 +71,10 @@ kit boot-fake-node --runtime-path ~/git/kinode ``` `kit` also contains tools for running tests. -For details and examples, please see [https://github.com/kinode-dao/core_tests](https://github.com/kinode-dao/core_tests). +For details and examples, please see +1. [Kinode Book's example code](https://github.com/kinode-dao/kinode-book/tree/main/code). +2. `kit`s templates, available through `kit new` or [here](https://github.com/kinode-dao/kit/tree/master/src/new/templates/rust). +3. [https://github.com/kinode-dao/core_tests](https://github.com/kinode-dao/core_tests). ## UI Development From 63f104771eb169fe2235a80b69f72e3295ad40d0 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 15 Nov 2024 11:14:31 -0800 Subject: [PATCH 07/17] new: update api wit file names --- ...{chat_test:template.os-v0.wit => chat-test:template.os-v0.wit} | 0 ...{echo_test:template.os-v0.wit => echo-test:template.os-v0.wit} | 0 ..._test:template.os-v0.wit => fibonacci-test:template.os-v0.wit} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename src/new/templates/rust/no-ui/chat/test/chat-test/api/{chat_test:template.os-v0.wit => chat-test:template.os-v0.wit} (100%) rename src/new/templates/rust/no-ui/echo/test/echo-test/api/{echo_test:template.os-v0.wit => echo-test:template.os-v0.wit} (100%) rename src/new/templates/rust/no-ui/fibonacci/test/fibonacci-test/api/{fibonacci_test:template.os-v0.wit => fibonacci-test:template.os-v0.wit} (100%) diff --git a/src/new/templates/rust/no-ui/chat/test/chat-test/api/chat_test:template.os-v0.wit b/src/new/templates/rust/no-ui/chat/test/chat-test/api/chat-test:template.os-v0.wit similarity index 100% rename from src/new/templates/rust/no-ui/chat/test/chat-test/api/chat_test:template.os-v0.wit rename to src/new/templates/rust/no-ui/chat/test/chat-test/api/chat-test:template.os-v0.wit diff --git a/src/new/templates/rust/no-ui/echo/test/echo-test/api/echo_test:template.os-v0.wit b/src/new/templates/rust/no-ui/echo/test/echo-test/api/echo-test:template.os-v0.wit similarity index 100% rename from src/new/templates/rust/no-ui/echo/test/echo-test/api/echo_test:template.os-v0.wit rename to src/new/templates/rust/no-ui/echo/test/echo-test/api/echo-test:template.os-v0.wit diff --git a/src/new/templates/rust/no-ui/fibonacci/test/fibonacci-test/api/fibonacci_test:template.os-v0.wit b/src/new/templates/rust/no-ui/fibonacci/test/fibonacci-test/api/fibonacci-test:template.os-v0.wit similarity index 100% rename from src/new/templates/rust/no-ui/fibonacci/test/fibonacci-test/api/fibonacci_test:template.os-v0.wit rename to src/new/templates/rust/no-ui/fibonacci/test/fibonacci-test/api/fibonacci-test:template.os-v0.wit From f8a8a1fad8ea14201742d3690eac465dc3443aec Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 15 Nov 2024 16:32:42 -0800 Subject: [PATCH 08/17] build: add minimum-viable rewrite-before-build --- src/build/mod.rs | 14 ++++++++-- src/build/rewrite.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 src/build/rewrite.rs diff --git a/src/build/mod.rs b/src/build/mod.rs index 1d19e8cc..6b5dc052 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -29,6 +29,9 @@ use crate::setup::{ use crate::view_api; use crate::KIT_CACHE; +mod rewrite; +use rewrite::copy_and_rewrite_package; + const PY_VENV_NAME: &str = "process_env"; const JAVASCRIPT_SRC_PATH: &str = "src/lib.js"; const PYTHON_SRC_PATH: &str = "src/lib.py"; @@ -1734,7 +1737,9 @@ pub async fn execute( check_process_lib_version(&package_dir.join("Cargo.toml"))?; - let ui_dirs = get_ui_dirs(package_dir, &include, &exclude)?; + let rewritten_dir = copy_and_rewrite_package(package_dir)?; + + let ui_dirs = get_ui_dirs(&rewritten_dir, &include, &exclude)?; if !no_ui && !ui_dirs.is_empty() { if !skip_deps_check { let mut recv_kill = make_fake_kill_chan(); @@ -1749,7 +1754,7 @@ pub async fn execute( if !ui_only { compile_package( - package_dir, + &rewritten_dir, skip_deps_check, features, url, @@ -1766,6 +1771,11 @@ pub async fn execute( .await?; } + if package_dir.join("pkg").exists() { + fs::remove_dir_all(package_dir.join("pkg"))?; + } + copy_dir(rewritten_dir.join("pkg"), package_dir.join("pkg"))?; + let metadata = read_metadata(package_dir)?; let pkg_publisher = make_pkg_publisher(&metadata); let (_zip_filename, hash_string) = zip_pkg(package_dir, &pkg_publisher)?; diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs new file mode 100644 index 00000000..4d0301d4 --- /dev/null +++ b/src/build/rewrite.rs @@ -0,0 +1,61 @@ +use std::path::{Path, PathBuf}; + +use color_eyre::Result; +use fs_err as fs; +use regex::Regex; +use tracing::{debug, instrument}; + +#[instrument(level = "trace", skip_all)] +pub fn copy_and_rewrite_package(package_dir: &Path) -> Result { + // Create target/rewrite/ directory + let rewrite_dir = package_dir.join("target").join("rewrite"); + if rewrite_dir.exists() { + fs::remove_dir_all(&rewrite_dir)?; + } + fs::create_dir_all(&rewrite_dir)?; + + // Copy package contents + copy_dir_and_rewrite(package_dir, &rewrite_dir)?; + + Ok(rewrite_dir) +} + +#[instrument(level = "trace", skip_all)] +fn copy_dir_and_rewrite(src: &Path, dst: &Path) -> Result<()> { + if !dst.exists() { + fs::create_dir_all(dst)?; + } + + for entry in fs::read_dir(src)? { + let entry = entry?; + let path = entry.path(); + let dest_path = dst.join(entry.file_name()); + + if path.is_dir() { + // Skip target/ directory to avoid recursion + if path.file_name().and_then(|n| n.to_str()) == Some("target") { + continue; + } + copy_dir_and_rewrite(&path, &dest_path)?; + } else { + if path.extension().and_then(|s| s.to_str()) == Some("rs") { + // Rewrite Rust files + let contents = fs::read_to_string(&path)?; + let new_contents = rewrite_rust_file(&contents)?; + debug!("rewrote {}", dest_path.display()); + fs::write(&dest_path, new_contents)?; + } else { + // Copy other files as-is + fs::copy(&path, &dest_path)?; + } + } + } + Ok(()) +} + +#[instrument(level = "trace", skip_all)] +fn rewrite_rust_file(content: &str) -> Result { + let println_re = Regex::new(r#"(\s*)println!\("(.*)"(.*)\)"#)?; + let result = println_re.replace_all(content, r#"${1}println!("hi ${2}"${3})"#); + Ok(result.into_owned()) +} From 3941cb9defbb81e0e1f27aa64dc84dad897940d1 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 19 Nov 2024 17:39:53 -0800 Subject: [PATCH 09/17] build: add minimum-viable rewrite of Spawn(|| {}) --- Cargo.lock | 2 + Cargo.toml | 2 + src/build/rewrite.rs | 455 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 435 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6770821..7bb2fccf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2175,8 +2175,10 @@ dependencies = [ "serde", "serde_json", "sha2", + "thiserror", "tokio", "toml", + "toml_edit 0.22.20", "tracing", "tracing-appender", "tracing-error", diff --git a/Cargo.toml b/Cargo.toml index cad4c00f..11bbb8ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10.8" +thiserror = "1.0" tokio = { version = "1.28", features = [ "macros", "process", @@ -55,6 +56,7 @@ tokio = { version = "1.28", features = [ "time", ] } toml = "0.8" +toml_edit = "0.22" tracing = "0.1" tracing-appender = "0.2" tracing-error = "0.2" diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs index 4d0301d4..4ac2fa2b 100644 --- a/src/build/rewrite.rs +++ b/src/build/rewrite.rs @@ -1,61 +1,468 @@ +use std::collections::HashMap; use std::path::{Path, PathBuf}; -use color_eyre::Result; +use color_eyre::{eyre::eyre, Result}; use fs_err as fs; use regex::Regex; +use toml_edit; use tracing::{debug, instrument}; +#[derive(Debug, Default)] +struct GeneratedProcesses { + // original process name -> (generated process name -> (wasm path, content)) + processes: HashMap>, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct GeneratedProcessesExternal { + // original process name -> (generated process name -> wasm path) + processes: HashMap>, +} + +impl From for GeneratedProcessesExternal { + fn from(input: GeneratedProcesses) -> Self { + let processes = input + .processes + .iter() + .map(|(parent_name, child_to_content)| { + ( + parent_name.to_string(), + child_to_content + .iter() + .map(|(child_name, (path, _content))| { + (child_name.to_string(), path.to_string()) + }) + .collect(), + ) + }) + .collect(); + GeneratedProcessesExternal { processes } + } +} + +#[derive(Debug)] +struct SpawnMatch { + args: String, + body: String, + imports: Vec, + start_pos: usize, + end_pos: usize, +} + +#[derive(Debug)] +struct SpawnInfo { + args: String, // The arguments passed to the spawn closure + body: String, // The body of the spawn closure + imports: Vec, // All imports from the original file + wit_bindgen: String, // `wit_bindgen!()` call +} + +#[derive(Debug, thiserror::Error)] +enum SpawnParseError { + #[error("Parse failed due to malformed imports")] + Imports, + #[error("Spawn parse failed due to malformed closure: no closing pipe in closure")] + NoClosingPipe, + #[error("Spawn parse failed due to malformed closure: no opening brace")] + NoOpeningBrace, + #[error("Spawn parse failed due to malformed closure: unclosed brace")] + UnclosedBrace, + #[error("Spawn parse failed due to malformed closure: unclosed paren")] + UnclosedParen, +} + +fn extract_imports(content: &str) -> Result, SpawnParseError> { + let imports_re = Regex::new(r"use\s+([^;]+);").map_err(|_| SpawnParseError::Imports)?; + Ok(imports_re + .captures_iter(content) + .map(|cap| cap[1].trim().to_string()) + .collect()) +} + +fn extract_wit_bindgen(content: &str) -> Option { + // Look for wit_bindgen::generate! macro + if let Some(start) = content.find("wit_bindgen::generate!") { + let mut brace_count = 0; + let mut in_macro = false; + let mut saw_closing_brace = false; + let mut saw_closing_paren = false; + let mut macro_end = start; + + // Find the closing part of the macro by counting braces + for (i, c) in content[start..].chars().enumerate() { + match c { + '{' => { + brace_count += 1; + in_macro = true; + } + '}' => { + brace_count -= 1; + if in_macro && brace_count == 0 { + saw_closing_brace = true; + } + } + ')' => { + if in_macro && saw_closing_brace && brace_count == 0 { + saw_closing_paren = true; + } + } + ';' => { + if in_macro && saw_closing_brace && saw_closing_paren && brace_count == 0 { + macro_end = start + i + 1; + break; + } + } + _ => {} + } + } + + Some(content[start..macro_end].to_string()) + } else { + None + } +} + +fn parse_spawn_from(input: &str) -> Result<(String, String, usize), SpawnParseError> { + // Skip the "Spawn(|" prefix since we know it's there + let input_after_spawn = &input["Spawn(|".len()..]; + + // Find the closing "|" + let pipe_end = input_after_spawn + .find('|') + .ok_or(SpawnParseError::NoClosingPipe)?; + + // Find the opening "{" + let brace_start = input_after_spawn[pipe_end..] + .find('{') + .ok_or(SpawnParseError::NoOpeningBrace)? + .saturating_add(pipe_end); + + // Find the closing "}" while handling nested braces + let mut brace_count = 1; + let mut brace_end = None; + let mut paren_end = None; + + for (i, c) in input_after_spawn[brace_start + 1..].chars().enumerate() { + match c { + '{' => brace_count += 1, + '}' => { + brace_count -= 1; + if brace_count == 0 { + brace_end = Some(brace_start + 1 + i); + } + } + ')' => { + if brace_count == 0 && brace_end.is_some() { + paren_end = Some(brace_start + 1 + i); + break; + } + } + _ => {} + } + } + + let brace_end = brace_end.ok_or(SpawnParseError::UnclosedBrace)?; + let paren_end = paren_end.ok_or(SpawnParseError::UnclosedParen)?; + + let args = input_after_spawn[..pipe_end].trim().to_string(); + let body = input_after_spawn[brace_start + 1..brace_end] + .trim() + .to_string(); + + // Return the total length consumed so we know where to continue searching + let total_consumed = "Spawn(|".len() + paren_end + 1; + + Ok((args, body, total_consumed)) +} + +fn find_all_spawns(input: &str) -> Result, SpawnParseError> { + let mut results = Vec::new(); + let mut search_from = 0; + let imports = extract_imports(input)?; + + while let Some(spawn_start) = input[search_from..].find("Spawn(|") { + let absolute_start = search_from + spawn_start; + + let (args, body, consumed_len) = parse_spawn_from(&input[absolute_start..])?; + + results.push(SpawnMatch { + args, + body, + imports: imports.clone(), + start_pos: absolute_start, + end_pos: absolute_start + consumed_len, + }); + + search_from = absolute_start + consumed_len; + } + + Ok(results) +} + +#[instrument(level = "trace", skip_all)] +fn generate_worker_process(process_name: &str, spawn_info: &SpawnInfo) -> Result { + let template = format!( + r#"// Generated worker process for {process_name} +{} + +{} + +call_init!(init); +fn init(our: Address) {{ + // Get args from parent + let message = await_message().expect("Failed to get args from parent"); + let args: serde_json::Value = serde_json::from_slice(&message.body()).unwrap(); + + // Execute original spawn body + {} + + // Exit after completion + std::process::exit(0); +}} +"#, + // Add all the original imports + spawn_info + .imports + .iter() + .map(|i| format!("use {i};\n")) + .collect::(), + spawn_info.wit_bindgen, + spawn_info.body + ); + + Ok(template) +} + #[instrument(level = "trace", skip_all)] pub fn copy_and_rewrite_package(package_dir: &Path) -> Result { - // Create target/rewrite/ directory + debug!("Rewriting for {}...", package_dir.display()); let rewrite_dir = package_dir.join("target").join("rewrite"); if rewrite_dir.exists() { fs::remove_dir_all(&rewrite_dir)?; } fs::create_dir_all(&rewrite_dir)?; - // Copy package contents - copy_dir_and_rewrite(package_dir, &rewrite_dir)?; + copy_dir(package_dir, &rewrite_dir)?; + + let mut generated = GeneratedProcesses::default(); + + // Process all Rust files in the copied directory + process_package(&rewrite_dir, &mut generated)?; + + // Create child processes + create_child_processes(&rewrite_dir, &generated)?; + + // Update workspace Cargo.toml + update_workspace_cargo_toml(&rewrite_dir, &generated)?; Ok(rewrite_dir) } +// TODO: factor out with build::mod.rs::copy_dir() #[instrument(level = "trace", skip_all)] -fn copy_dir_and_rewrite(src: &Path, dst: &Path) -> Result<()> { +fn copy_dir(src: impl AsRef, dst: impl AsRef) -> Result<()> { + let src = src.as_ref(); + let dst = dst.as_ref(); if !dst.exists() { fs::create_dir_all(dst)?; } for entry in fs::read_dir(src)? { let entry = entry?; - let path = entry.path(); - let dest_path = dst.join(entry.file_name()); + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); - if path.is_dir() { - // Skip target/ directory to avoid recursion - if path.file_name().and_then(|n| n.to_str()) == Some("target") { + if src_path.is_dir() { + if src_path.file_name().and_then(|n| n.to_str()) == Some("target") { continue; } - copy_dir_and_rewrite(&path, &dest_path)?; + copy_dir(&src_path, &dst_path)?; } else { - if path.extension().and_then(|s| s.to_str()) == Some("rs") { - // Rewrite Rust files - let contents = fs::read_to_string(&path)?; - let new_contents = rewrite_rust_file(&contents)?; - debug!("rewrote {}", dest_path.display()); - fs::write(&dest_path, new_contents)?; - } else { - // Copy other files as-is - fs::copy(&path, &dest_path)?; + fs::copy(&src_path, &dst_path)?; + } + } + Ok(()) +} + +#[instrument(level = "trace", skip_all)] +fn create_child_processes(package_dir: &Path, generated: &GeneratedProcesses) -> Result<()> { + for (process_name, workers) in &generated.processes { + for (worker_name, (_, content)) in workers { + let parent_dir = package_dir.join(process_name); + let worker_dir = package_dir.join(worker_name); + + // Copy the source directory structure from parent + let parent_src = parent_dir.join("src"); + let worker_src = worker_dir.join("src"); + debug!("{} {}", parent_src.display(), worker_src.display()); + copy_dir(&parent_src, &worker_src)?; + + // Overwrite lib.rs with our generated content + fs::write(worker_src.join("lib.rs"), content)?; + + // Copy and modify Cargo.toml + let parent_cargo = fs::read_to_string(parent_dir.join("Cargo.toml"))?; + let mut doc = parent_cargo.parse::()?; + + // Update package name to worker name + if let Some(package) = doc.get_mut("package") { + if let Some(name) = package.get_mut("name") { + *name = toml_edit::value(worker_name.as_str()); + } } + + fs::write(worker_dir.join("Cargo.toml"), doc.to_string())?; + } + } + Ok(()) +} + +#[instrument(level = "trace", skip_all)] +fn update_workspace_cargo_toml(package_dir: &Path, generated: &GeneratedProcesses) -> Result<()> { + let cargo_toml_path = package_dir.join("Cargo.toml"); + let cargo_toml = fs::read_to_string(&cargo_toml_path)?; + + // Parse existing TOML + let mut doc = cargo_toml.parse::()?; + + // Get or create workspace section + let workspace = doc.entry("workspace").or_insert(toml_edit::table()); + + // Get or create members array + let members = workspace + .as_table_mut() + .ok_or_else(|| eyre!("workspace is not a table"))? + .entry("members") + .or_insert(toml_edit::array()); + + let members_array = members + .as_array_mut() + .ok_or_else(|| eyre!("members is not an array"))?; + + // Add all worker packages + for workers in generated.processes.values() { + for worker_name in workers.keys() { + members_array.push(worker_name); } } + + // Write back to file + fs::write(cargo_toml_path, doc.to_string())?; + Ok(()) } #[instrument(level = "trace", skip_all)] -fn rewrite_rust_file(content: &str) -> Result { - let println_re = Regex::new(r#"(\s*)println!\("(.*)"(.*)\)"#)?; - let result = println_re.replace_all(content, r#"${1}println!("hi ${2}"${3})"#); - Ok(result.into_owned()) +fn rewrite_rust_file( + process_name: &str, + file_name: &str, + content: &str, + generated: &mut GeneratedProcesses, +) -> Result { + let spawn_matches = find_all_spawns(content)?; + let mut new_content = content.to_string(); + + // Process spawns in reverse order to not invalidate positions + for (i, spawn_match) in spawn_matches.iter().enumerate().rev() { + let worker_name = format!("{process_name}-worker-{i}"); + let wasm_name = format!("{worker_name}.wasm"); + + // Generate worker process + let wit_bindgen = extract_wit_bindgen(content).unwrap_or_else(|| { + // Fallback to default if not found + r#"wit_bindgen::generate!({ + path: "target/wit", + world: "process-v0", +})"# + .to_string() + }); + + let worker_code = generate_worker_process( + file_name, + &SpawnInfo { + args: spawn_match.args.clone(), + body: spawn_match.body.clone(), + imports: spawn_match.imports.clone(), + wit_bindgen, + }, + )?; + + // Track in generated processes + generated + .processes + .entry(process_name.to_string()) + .or_default() + .insert(worker_name.clone(), (wasm_name, worker_code)); + + // Create replacement spawn code + let args = spawn_match + .args + .split(", ") + .map(|s| format!("\"{s}\":{s}")) + .collect::>() + .join(","); + let args = "{".to_string() + &args; + let args = args + "}"; + let replacement = format!( + r#"{{ + use kinode_process_lib::{{spawn, OnExit, Request}}; + let worker = spawn( + None, + &format!("{{}}:{{}}/pkg/{}.wasm", our.process.package_name, our.process.publisher_node), + OnExit::None, + vec![], + vec![], + false, + ).expect("failed to spawn worker"); + Request::to((our.node(), worker)) + .body(serde_json::to_vec(&serde_json::json!({})).unwrap()) + .send() + .expect("failed to initialize worker"); + }}"#, + worker_name, args, + ); + + // Replace in the content using positions + new_content.replace_range(spawn_match.start_pos..spawn_match.end_pos, &replacement); + } + + Ok(new_content) +} + +#[instrument(level = "trace", skip_all)] +fn process_package(package_dir: &Path, generated: &mut GeneratedProcesses) -> Result<()> { + if !package_dir.is_dir() { + return Ok(()); + } + + for entry in fs::read_dir(package_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + if path.file_name().and_then(|n| n.to_str()) == Some("target") { + continue; + } + process_package(&path, generated)?; + } else if path.extension().and_then(|s| s.to_str()) == Some("rs") { + let process_name = path + .parent() + .and_then(|p| p.parent()) + .and_then(|n| n.file_name()) + .and_then(|n| n.to_str()) + .ok_or_else(|| eyre!("Invalid process name"))? + .to_string(); + + let file_name = path + .file_stem() + .and_then(|n| n.to_str()) + .ok_or_else(|| eyre!("Invalid file name"))? + .to_string(); + + let content = fs::read_to_string(&path)?; + let new_content = rewrite_rust_file(&process_name, &file_name, &content, generated)?; + fs::write(&path, new_content)?; + } + } + Ok(()) } From b548cb69a9600aef33e32d388e13c80174d611df Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 20 Nov 2024 11:00:11 -0800 Subject: [PATCH 10/17] build: rewrite `Spawn!()` rather than `Spawn()` for happier linter --- src/build/rewrite.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs index 4ac2fa2b..f6554133 100644 --- a/src/build/rewrite.rs +++ b/src/build/rewrite.rs @@ -123,8 +123,8 @@ fn extract_wit_bindgen(content: &str) -> Option { } fn parse_spawn_from(input: &str) -> Result<(String, String, usize), SpawnParseError> { - // Skip the "Spawn(|" prefix since we know it's there - let input_after_spawn = &input["Spawn(|".len()..]; + // Skip the "Spawn!(|" prefix since we know it's there + let input_after_spawn = &input["Spawn!(|".len()..]; // Find the closing "|" let pipe_end = input_after_spawn @@ -170,7 +170,7 @@ fn parse_spawn_from(input: &str) -> Result<(String, String, usize), SpawnParseEr .to_string(); // Return the total length consumed so we know where to continue searching - let total_consumed = "Spawn(|".len() + paren_end + 1; + let total_consumed = "Spawn!(|".len() + paren_end + 1; Ok((args, body, total_consumed)) } @@ -180,7 +180,7 @@ fn find_all_spawns(input: &str) -> Result, SpawnParseError> { let mut search_from = 0; let imports = extract_imports(input)?; - while let Some(spawn_start) = input[search_from..].find("Spawn(|") { + while let Some(spawn_start) = input[search_from..].find("Spawn!(|") { let absolute_start = search_from + spawn_start; let (args, body, consumed_len) = parse_spawn_from(&input[absolute_start..])?; From 26ab5fa45cc6446a16246888c6103c313fd023b2 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 20 Nov 2024 11:39:46 -0800 Subject: [PATCH 11/17] build: make rewriting optional, but default, with `--no-rewrite` flag --- src/build/mod.rs | 28 ++++++++++++++++++++++------ src/build_start_package/mod.rs | 2 ++ src/main.rs | 16 ++++++++++++++++ src/run_tests/mod.rs | 3 +++ 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 6b5dc052..5330cc5e 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1158,6 +1158,7 @@ async fn fetch_dependencies( default_world: Option<&str>, include: &HashSet, exclude: &HashSet, + no_rewrite: bool, force: bool, verbose: bool, ) -> Result<()> { @@ -1174,6 +1175,7 @@ async fn fetch_dependencies( default_world, vec![], // TODO: what about deps-of-deps? vec![], + no_rewrite, false, force, verbose, @@ -1210,6 +1212,7 @@ async fn fetch_dependencies( default_world, local_dep_deps, vec![], + no_rewrite, false, force, verbose, @@ -1525,6 +1528,7 @@ async fn compile_package( add_paths_to_api: &Vec, include: &HashSet, exclude: &HashSet, + no_rewrite: bool, force: bool, verbose: bool, ignore_deps: bool, // for internal use; may cause problems when adding recursive deps @@ -1547,6 +1551,7 @@ async fn compile_package( default_world, include, exclude, + no_rewrite, force, verbose, ) @@ -1654,6 +1659,7 @@ pub async fn execute( default_world: Option<&str>, local_dependencies: Vec, add_paths_to_api: Vec, + no_rewrite: bool, reproducible: bool, force: bool, verbose: bool, @@ -1737,9 +1743,16 @@ pub async fn execute( check_process_lib_version(&package_dir.join("Cargo.toml"))?; - let rewritten_dir = copy_and_rewrite_package(package_dir)?; + // live_dir is the "dir that is being built" or is "live"; + // if `no_rewrite`, that is just `package_dir`; + // else, it is the modified copy that is in `target/rewrite/` + let live_dir = if no_rewrite { + PathBuf::from(package_dir) + } else { + copy_and_rewrite_package(package_dir)? + }; - let ui_dirs = get_ui_dirs(&rewritten_dir, &include, &exclude)?; + let ui_dirs = get_ui_dirs(&live_dir, &include, &exclude)?; if !no_ui && !ui_dirs.is_empty() { if !skip_deps_check { let mut recv_kill = make_fake_kill_chan(); @@ -1754,7 +1767,7 @@ pub async fn execute( if !ui_only { compile_package( - &rewritten_dir, + &live_dir, skip_deps_check, features, url, @@ -1764,6 +1777,7 @@ pub async fn execute( &add_paths_to_api, &include, &exclude, + no_rewrite, force, verbose, ignore_deps, @@ -1771,10 +1785,12 @@ pub async fn execute( .await?; } - if package_dir.join("pkg").exists() { - fs::remove_dir_all(package_dir.join("pkg"))?; + if !no_rewrite { + if package_dir.join("pkg").exists() { + fs::remove_dir_all(package_dir.join("pkg"))?; + } + copy_dir(live_dir.join("pkg"), package_dir.join("pkg"))?; } - copy_dir(rewritten_dir.join("pkg"), package_dir.join("pkg"))?; let metadata = read_metadata(package_dir)?; let pkg_publisher = make_pkg_publisher(&metadata); diff --git a/src/build_start_package/mod.rs b/src/build_start_package/mod.rs index 2a29546a..36eec647 100644 --- a/src/build_start_package/mod.rs +++ b/src/build_start_package/mod.rs @@ -21,6 +21,7 @@ pub async fn execute( default_world: Option<&str>, local_dependencies: Vec, add_paths_to_api: Vec, + no_rewrite: bool, reproducible: bool, force: bool, verbose: bool, @@ -38,6 +39,7 @@ pub async fn execute( default_world, local_dependencies, add_paths_to_api, + no_rewrite, reproducible, force, verbose, diff --git a/src/main.rs b/src/main.rs index f111bfc4..c5d1892f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -222,6 +222,7 @@ async fn execute( .unwrap_or_default() .map(|s| PathBuf::from(s)) .collect(); + let no_rewrite = matches.get_one::("NO_REWRITE").unwrap(); let reproducible = matches.get_one::("REPRODUCIBLE").unwrap(); let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); @@ -239,6 +240,7 @@ async fn execute( default_world.map(|w| w.as_str()), local_dependencies, add_paths_to_api, + *no_rewrite, *reproducible, *force, *verbose, @@ -283,6 +285,7 @@ async fn execute( .unwrap_or_default() .map(|s| PathBuf::from(s)) .collect(); + let no_rewrite = matches.get_one::("NO_REWRITE").unwrap(); let reproducible = matches.get_one::("REPRODUCIBLE").unwrap(); let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); @@ -300,6 +303,7 @@ async fn execute( default_world.map(|w| w.as_str()), local_dependencies, add_paths_to_api, + *no_rewrite, *reproducible, *force, *verbose, @@ -733,6 +737,12 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .long("add-to-api") .help("Path to file to add to api.zip (can specify multiple times)") ) + .arg(Arg::new("NO_REWRITE") + .action(ArgAction::SetTrue) + .long("no-rewrite") + .help("Don't rewrite the package (disables `Spawn!()`) [default: rewrite]") + .required(false) + ) .arg(Arg::new("REPRODUCIBLE") .action(ArgAction::SetTrue) .short('r') @@ -834,6 +844,12 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Pass these comma-delimited feature flags to Rust cargo builds") .required(false) ) + .arg(Arg::new("NO_REWRITE") + .action(ArgAction::SetTrue) + .long("no-rewrite") + .help("Don't rewrite the package (disables `Spawn!()`) [default: rewrite]") + .required(false) + ) .arg(Arg::new("REPRODUCIBLE") .action(ArgAction::SetTrue) .short('r') diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index 22079033..94b9a3b0 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -382,6 +382,7 @@ async fn build_packages( false, false, false, + false, ) .await?; debug!("Start {path:?}"); @@ -406,6 +407,7 @@ async fn build_packages( false, false, false, + false, ) .await?; } @@ -427,6 +429,7 @@ async fn build_packages( false, false, false, + false, ) .await?; } From 5434390bea4cf63b421710bdc5140af46b3154ca Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 20 Nov 2024 17:29:57 -0800 Subject: [PATCH 12/17] build: improve arg passing with struct; destructure into child for max devex --- Cargo.lock | 1 + Cargo.toml | 1 + src/build/rewrite.rs | 169 ++++++++++++++++++++++++++++++------------- src/new/mod.rs | 2 +- 4 files changed, 121 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bb2fccf..055174b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2175,6 +2175,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "syn 2.0.75", "thiserror", "tokio", "toml", diff --git a/Cargo.toml b/Cargo.toml index 11bbb8ed..48a62ed5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10.8" +syn = { version = "2.0", features = ["full"] } thiserror = "1.0" tokio = { version = "1.28", features = [ "macros", diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs index f6554133..2fa4f44a 100644 --- a/src/build/rewrite.rs +++ b/src/build/rewrite.rs @@ -4,9 +4,12 @@ use std::path::{Path, PathBuf}; use color_eyre::{eyre::eyre, Result}; use fs_err as fs; use regex::Regex; +use syn::{__private::ToTokens, parse_str}; use toml_edit; use tracing::{debug, instrument}; +use crate::new::snake_to_upper_camel_case; + #[derive(Debug, Default)] struct GeneratedProcesses { // original process name -> (generated process name -> (wasm path, content)) @@ -40,6 +43,12 @@ impl From for GeneratedProcessesExternal { } } +#[derive(Debug)] +struct ArgInfo { + name: String, + ty: String, +} + #[derive(Debug)] struct SpawnMatch { args: String, @@ -49,14 +58,6 @@ struct SpawnMatch { end_pos: usize, } -#[derive(Debug)] -struct SpawnInfo { - args: String, // The arguments passed to the spawn closure - body: String, // The body of the spawn closure - imports: Vec, // All imports from the original file - wit_bindgen: String, // `wit_bindgen!()` call -} - #[derive(Debug, thiserror::Error)] enum SpawnParseError { #[error("Parse failed due to malformed imports")] @@ -71,6 +72,77 @@ enum SpawnParseError { UnclosedParen, } +#[instrument(level = "trace", skip_all)] +fn parse_fn_args(args: &str) -> Result> { + // Parse the argument string as Rust function parameters + let fn_item: syn::ItemFn = parse_str(&format!("fn dummy({args}) {{}}"))?; + + // Extract the parameters from the function signature + let params = fn_item + .sig + .inputs + .into_iter() + .filter_map(|param| { + if let syn::FnArg::Typed(pat_type) = param { + Some(ArgInfo { + name: pat_type.pat.into_token_stream().to_string(), + ty: pat_type.ty.into_token_stream().to_string(), + }) + } else { + None + } + }) + .collect(); + + Ok(params) +} + +fn make_args_struct_name(worker_name: &str) -> String { + format!( + "{}Args", + snake_to_upper_camel_case(&worker_name.replace("-", "_")) + ) +} + +fn generate_args_struct_type(struct_name: &str, args: &[ArgInfo]) -> String { + let fields = args + .iter() + .map(|arg| format!(" {}: {},", arg.name, arg.ty)) + .collect::>() + .join("\n"); + + format!( + r#"#[derive(serde::Serialize, serde::Deserialize)] +struct {struct_name} {{ +{fields} +}}"# + ) +} + +fn generate_args_struct_instance(struct_name: &str, args: &[ArgInfo]) -> String { + let fields = args + .iter() + .map(|arg| format!(" {0}: {0}.clone(),", arg.name)) + .collect::>() + .join("\n"); + + format!( + r#"let args = {struct_name} {{ +{fields} + }};"# + ) +} + +fn generate_args_struct_destructure(struct_name: &str, args: &[ArgInfo]) -> String { + let fields = args + .iter() + .map(|arg| arg.name.clone()) + .collect::>() + .join(", "); + + format!(r#"let {struct_name} {{ {fields} }}"#) +} + fn extract_imports(content: &str) -> Result, SpawnParseError> { let imports_re = Regex::new(r"use\s+([^;]+);").map_err(|_| SpawnParseError::Imports)?; Ok(imports_re @@ -200,34 +272,36 @@ fn find_all_spawns(input: &str) -> Result, SpawnParseError> { } #[instrument(level = "trace", skip_all)] -fn generate_worker_process(process_name: &str, spawn_info: &SpawnInfo) -> Result { +fn generate_worker_process( + process_name: &str, + body: &str, + imports: &[String], + wit_bindgen: &str, + args_type: &str, + args_destructure: &str, +) -> Result { + let imports = imports + .iter() + .map(|i| format!("#[allow(unused_imports)]\nuse {i};\n")) + .collect::(); let template = format!( r#"// Generated worker process for {process_name} -{} +{imports} + +{wit_bindgen} -{} +{args_type} call_init!(init); fn init(our: Address) {{ // Get args from parent let message = await_message().expect("Failed to get args from parent"); - let args: serde_json::Value = serde_json::from_slice(&message.body()).unwrap(); + {args_destructure} = serde_json::from_slice(&message.body()).unwrap(); - // Execute original spawn body - {} - - // Exit after completion - std::process::exit(0); + // Execute `Spawn!()` function body + {body} }} "#, - // Add all the original imports - spawn_info - .imports - .iter() - .map(|i| format!("use {i};\n")) - .collect::(), - spawn_info.wit_bindgen, - spawn_info.body ); Ok(template) @@ -355,7 +429,6 @@ fn update_workspace_cargo_toml(package_dir: &Path, generated: &GeneratedProcesse #[instrument(level = "trace", skip_all)] fn rewrite_rust_file( process_name: &str, - file_name: &str, content: &str, generated: &mut GeneratedProcesses, ) -> Result { @@ -367,6 +440,12 @@ fn rewrite_rust_file( let worker_name = format!("{process_name}-worker-{i}"); let wasm_name = format!("{worker_name}.wasm"); + let args_name = make_args_struct_name(&worker_name); + let parsed_args = parse_fn_args(&spawn_match.args)?; + let args_type = generate_args_struct_type(&args_name, &parsed_args); + let args_instance = generate_args_struct_instance(&args_name, &parsed_args); + let args_destructure = generate_args_struct_destructure(&args_name, &parsed_args); + // Generate worker process let wit_bindgen = extract_wit_bindgen(content).unwrap_or_else(|| { // Fallback to default if not found @@ -378,13 +457,12 @@ fn rewrite_rust_file( }); let worker_code = generate_worker_process( - file_name, - &SpawnInfo { - args: spawn_match.args.clone(), - body: spawn_match.body.clone(), - imports: spawn_match.imports.clone(), - wit_bindgen, - }, + process_name, + &spawn_match.body, + &spawn_match.imports, + &wit_bindgen, + &args_type, + &args_destructure, )?; // Track in generated processes @@ -395,31 +473,26 @@ fn rewrite_rust_file( .insert(worker_name.clone(), (wasm_name, worker_code)); // Create replacement spawn code - let args = spawn_match - .args - .split(", ") - .map(|s| format!("\"{s}\":{s}")) - .collect::>() - .join(","); - let args = "{".to_string() + &args; - let args = args + "}"; let replacement = format!( r#"{{ use kinode_process_lib::{{spawn, OnExit, Request}}; + {args_type} + + {args_instance} + let worker = spawn( None, - &format!("{{}}:{{}}/pkg/{}.wasm", our.process.package_name, our.process.publisher_node), + &format!("{{}}:{{}}/pkg/{worker_name}.wasm", our.process.package_name, our.process.publisher_node), OnExit::None, vec![], vec![], false, ).expect("failed to spawn worker"); Request::to((our.node(), worker)) - .body(serde_json::to_vec(&serde_json::json!({})).unwrap()) + .body(serde_json::to_vec(&args).unwrap()) .send() .expect("failed to initialize worker"); }}"#, - worker_name, args, ); // Replace in the content using positions @@ -453,14 +526,8 @@ fn process_package(package_dir: &Path, generated: &mut GeneratedProcesses) -> Re .ok_or_else(|| eyre!("Invalid process name"))? .to_string(); - let file_name = path - .file_stem() - .and_then(|n| n.to_str()) - .ok_or_else(|| eyre!("Invalid file name"))? - .to_string(); - let content = fs::read_to_string(&path)?; - let new_content = rewrite_rust_file(&process_name, &file_name, &content, generated)?; + let new_content = rewrite_rust_file(&process_name, &content, generated)?; fs::write(&path, new_content)?; } } diff --git a/src/new/mod.rs b/src/new/mod.rs index 681cbf41..60226f47 100644 --- a/src/new/mod.rs +++ b/src/new/mod.rs @@ -73,7 +73,7 @@ impl From<&String> for Template { } } -fn snake_to_upper_camel_case(input: &str) -> String { +pub fn snake_to_upper_camel_case(input: &str) -> String { let parts: Vec<&str> = input.split('_').collect(); let mut camel_case = String::new(); From 69bc1e834c383c8fe5420eae69351cce5ad1fe49 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 20 Nov 2024 17:30:09 -0800 Subject: [PATCH 13/17] build: restructure rewrite (pure refactor) --- src/build/rewrite.rs | 284 +++++++++++++++++++++---------------------- 1 file changed, 142 insertions(+), 142 deletions(-) diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs index 2fa4f44a..34b36df6 100644 --- a/src/build/rewrite.rs +++ b/src/build/rewrite.rs @@ -72,6 +72,39 @@ enum SpawnParseError { UnclosedParen, } +// TODO: factor out with build::mod.rs::copy_dir() +#[instrument(level = "trace", skip_all)] +fn copy_dir(src: impl AsRef, dst: impl AsRef) -> Result<()> { + let src = src.as_ref(); + let dst = dst.as_ref(); + if !dst.exists() { + fs::create_dir_all(dst)?; + } + + for entry in fs::read_dir(src)? { + let entry = entry?; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + + if src_path.is_dir() { + if src_path.file_name().and_then(|n| n.to_str()) == Some("target") { + continue; + } + copy_dir(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path)?; + } + } + Ok(()) +} + +fn make_args_struct_name(worker_name: &str) -> String { + format!( + "{}Args", + snake_to_upper_camel_case(&worker_name.replace("-", "_")) + ) +} + #[instrument(level = "trace", skip_all)] fn parse_fn_args(args: &str) -> Result> { // Parse the argument string as Rust function parameters @@ -97,13 +130,6 @@ fn parse_fn_args(args: &str) -> Result> { Ok(params) } -fn make_args_struct_name(worker_name: &str) -> String { - format!( - "{}Args", - snake_to_upper_camel_case(&worker_name.replace("-", "_")) - ) -} - fn generate_args_struct_type(struct_name: &str, args: &[ArgInfo]) -> String { let fields = args .iter() @@ -247,30 +273,6 @@ fn parse_spawn_from(input: &str) -> Result<(String, String, usize), SpawnParseEr Ok((args, body, total_consumed)) } -fn find_all_spawns(input: &str) -> Result, SpawnParseError> { - let mut results = Vec::new(); - let mut search_from = 0; - let imports = extract_imports(input)?; - - while let Some(spawn_start) = input[search_from..].find("Spawn!(|") { - let absolute_start = search_from + spawn_start; - - let (args, body, consumed_len) = parse_spawn_from(&input[absolute_start..])?; - - results.push(SpawnMatch { - args, - body, - imports: imports.clone(), - start_pos: absolute_start, - end_pos: absolute_start + consumed_len, - }); - - search_from = absolute_start + consumed_len; - } - - Ok(results) -} - #[instrument(level = "trace", skip_all)] fn generate_worker_process( process_name: &str, @@ -307,123 +309,28 @@ fn init(our: Address) {{ Ok(template) } -#[instrument(level = "trace", skip_all)] -pub fn copy_and_rewrite_package(package_dir: &Path) -> Result { - debug!("Rewriting for {}...", package_dir.display()); - let rewrite_dir = package_dir.join("target").join("rewrite"); - if rewrite_dir.exists() { - fs::remove_dir_all(&rewrite_dir)?; - } - fs::create_dir_all(&rewrite_dir)?; - - copy_dir(package_dir, &rewrite_dir)?; - - let mut generated = GeneratedProcesses::default(); - - // Process all Rust files in the copied directory - process_package(&rewrite_dir, &mut generated)?; - - // Create child processes - create_child_processes(&rewrite_dir, &generated)?; - - // Update workspace Cargo.toml - update_workspace_cargo_toml(&rewrite_dir, &generated)?; - - Ok(rewrite_dir) -} - -// TODO: factor out with build::mod.rs::copy_dir() -#[instrument(level = "trace", skip_all)] -fn copy_dir(src: impl AsRef, dst: impl AsRef) -> Result<()> { - let src = src.as_ref(); - let dst = dst.as_ref(); - if !dst.exists() { - fs::create_dir_all(dst)?; - } - - for entry in fs::read_dir(src)? { - let entry = entry?; - let src_path = entry.path(); - let dst_path = dst.join(entry.file_name()); - - if src_path.is_dir() { - if src_path.file_name().and_then(|n| n.to_str()) == Some("target") { - continue; - } - copy_dir(&src_path, &dst_path)?; - } else { - fs::copy(&src_path, &dst_path)?; - } - } - Ok(()) -} - -#[instrument(level = "trace", skip_all)] -fn create_child_processes(package_dir: &Path, generated: &GeneratedProcesses) -> Result<()> { - for (process_name, workers) in &generated.processes { - for (worker_name, (_, content)) in workers { - let parent_dir = package_dir.join(process_name); - let worker_dir = package_dir.join(worker_name); - - // Copy the source directory structure from parent - let parent_src = parent_dir.join("src"); - let worker_src = worker_dir.join("src"); - debug!("{} {}", parent_src.display(), worker_src.display()); - copy_dir(&parent_src, &worker_src)?; - - // Overwrite lib.rs with our generated content - fs::write(worker_src.join("lib.rs"), content)?; - - // Copy and modify Cargo.toml - let parent_cargo = fs::read_to_string(parent_dir.join("Cargo.toml"))?; - let mut doc = parent_cargo.parse::()?; - - // Update package name to worker name - if let Some(package) = doc.get_mut("package") { - if let Some(name) = package.get_mut("name") { - *name = toml_edit::value(worker_name.as_str()); - } - } - - fs::write(worker_dir.join("Cargo.toml"), doc.to_string())?; - } - } - Ok(()) -} - -#[instrument(level = "trace", skip_all)] -fn update_workspace_cargo_toml(package_dir: &Path, generated: &GeneratedProcesses) -> Result<()> { - let cargo_toml_path = package_dir.join("Cargo.toml"); - let cargo_toml = fs::read_to_string(&cargo_toml_path)?; - - // Parse existing TOML - let mut doc = cargo_toml.parse::()?; +fn find_all_spawns(input: &str) -> Result, SpawnParseError> { + let mut results = Vec::new(); + let mut search_from = 0; + let imports = extract_imports(input)?; - // Get or create workspace section - let workspace = doc.entry("workspace").or_insert(toml_edit::table()); + while let Some(spawn_start) = input[search_from..].find("Spawn!(|") { + let absolute_start = search_from + spawn_start; - // Get or create members array - let members = workspace - .as_table_mut() - .ok_or_else(|| eyre!("workspace is not a table"))? - .entry("members") - .or_insert(toml_edit::array()); + let (args, body, consumed_len) = parse_spawn_from(&input[absolute_start..])?; - let members_array = members - .as_array_mut() - .ok_or_else(|| eyre!("members is not an array"))?; + results.push(SpawnMatch { + args, + body, + imports: imports.clone(), + start_pos: absolute_start, + end_pos: absolute_start + consumed_len, + }); - // Add all worker packages - for workers in generated.processes.values() { - for worker_name in workers.keys() { - members_array.push(worker_name); - } + search_from = absolute_start + consumed_len; } - // Write back to file - fs::write(cargo_toml_path, doc.to_string())?; - - Ok(()) + Ok(results) } #[instrument(level = "trace", skip_all)] @@ -533,3 +440,96 @@ fn process_package(package_dir: &Path, generated: &mut GeneratedProcesses) -> Re } Ok(()) } + +#[instrument(level = "trace", skip_all)] +fn create_child_processes(package_dir: &Path, generated: &GeneratedProcesses) -> Result<()> { + for (process_name, workers) in &generated.processes { + for (worker_name, (_, content)) in workers { + let parent_dir = package_dir.join(process_name); + let worker_dir = package_dir.join(worker_name); + + // Copy the source directory structure from parent + let parent_src = parent_dir.join("src"); + let worker_src = worker_dir.join("src"); + debug!("{} {}", parent_src.display(), worker_src.display()); + copy_dir(&parent_src, &worker_src)?; + + // Overwrite lib.rs with our generated content + fs::write(worker_src.join("lib.rs"), content)?; + + // Copy and modify Cargo.toml + let parent_cargo = fs::read_to_string(parent_dir.join("Cargo.toml"))?; + let mut doc = parent_cargo.parse::()?; + + // Update package name to worker name + if let Some(package) = doc.get_mut("package") { + if let Some(name) = package.get_mut("name") { + *name = toml_edit::value(worker_name.as_str()); + } + } + + fs::write(worker_dir.join("Cargo.toml"), doc.to_string())?; + } + } + Ok(()) +} + +#[instrument(level = "trace", skip_all)] +fn update_workspace_cargo_toml(package_dir: &Path, generated: &GeneratedProcesses) -> Result<()> { + let cargo_toml_path = package_dir.join("Cargo.toml"); + let cargo_toml = fs::read_to_string(&cargo_toml_path)?; + + // Parse existing TOML + let mut doc = cargo_toml.parse::()?; + + // Get or create workspace section + let workspace = doc.entry("workspace").or_insert(toml_edit::table()); + + // Get or create members array + let members = workspace + .as_table_mut() + .ok_or_else(|| eyre!("workspace is not a table"))? + .entry("members") + .or_insert(toml_edit::array()); + + let members_array = members + .as_array_mut() + .ok_or_else(|| eyre!("members is not an array"))?; + + // Add all worker packages + for workers in generated.processes.values() { + for worker_name in workers.keys() { + members_array.push(worker_name); + } + } + + // Write back to file + fs::write(cargo_toml_path, doc.to_string())?; + + Ok(()) +} + +#[instrument(level = "trace", skip_all)] +pub fn copy_and_rewrite_package(package_dir: &Path) -> Result { + debug!("Rewriting for {}...", package_dir.display()); + let rewrite_dir = package_dir.join("target").join("rewrite"); + if rewrite_dir.exists() { + fs::remove_dir_all(&rewrite_dir)?; + } + fs::create_dir_all(&rewrite_dir)?; + + copy_dir(package_dir, &rewrite_dir)?; + + let mut generated = GeneratedProcesses::default(); + + // Process all Rust files in the copied directory + process_package(&rewrite_dir, &mut generated)?; + + // Create child processes + create_child_processes(&rewrite_dir, &generated)?; + + // Update workspace Cargo.toml + update_workspace_cargo_toml(&rewrite_dir, &generated)?; + + Ok(rewrite_dir) +} From 10a9440195994bce9b28006efa6b8bdf7492ec9b Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 20 Nov 2024 17:43:30 -0800 Subject: [PATCH 14/17] build: `rustfmt` the codegend files --- src/build/rewrite.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs index 34b36df6..33c30a12 100644 --- a/src/build/rewrite.rs +++ b/src/build/rewrite.rs @@ -333,6 +333,8 @@ fn find_all_spawns(input: &str) -> Result, SpawnParseError> { Ok(results) } +/// Rewrites the parent and stores information +/// for writing children in GeneratedProcess. #[instrument(level = "trace", skip_all)] fn rewrite_rust_file( process_name: &str, @@ -409,6 +411,8 @@ fn rewrite_rust_file( Ok(new_content) } +/// For each process in package, rewrite rust files parents +/// and store information for writing children in GeneratedProcess. #[instrument(level = "trace", skip_all)] fn process_package(package_dir: &Path, generated: &mut GeneratedProcesses) -> Result<()> { if !package_dir.is_dir() { @@ -436,6 +440,7 @@ fn process_package(package_dir: &Path, generated: &mut GeneratedProcesses) -> Re let content = fs::read_to_string(&path)?; let new_content = rewrite_rust_file(&process_name, &content, generated)?; fs::write(&path, new_content)?; + crate::build::run_command(std::process::Command::new("rustfmt").arg(&path), false)?; } } Ok(()) @@ -455,7 +460,12 @@ fn create_child_processes(package_dir: &Path, generated: &GeneratedProcesses) -> copy_dir(&parent_src, &worker_src)?; // Overwrite lib.rs with our generated content - fs::write(worker_src.join("lib.rs"), content)?; + let worker_lib = worker_src.join("lib.rs"); + fs::write(&worker_lib, content)?; + crate::build::run_command( + std::process::Command::new("rustfmt").arg(&worker_lib), + false, + )?; // Copy and modify Cargo.toml let parent_cargo = fs::read_to_string(parent_dir.join("Cargo.toml"))?; @@ -522,7 +532,7 @@ pub fn copy_and_rewrite_package(package_dir: &Path) -> Result { let mut generated = GeneratedProcesses::default(); - // Process all Rust files in the copied directory + // Rewrite parents & gather info for writing children process_package(&rewrite_dir, &mut generated)?; // Create child processes From 9bc0dcac164ffea74ddc09c6a4425d4891372248 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 21 Nov 2024 21:06:48 -0800 Subject: [PATCH 15/17] build: port functions to child when used; allow `Spawn!()` of function OR closure --- Cargo.lock | 1 + Cargo.toml | 4 +- src/build/rewrite.rs | 467 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 446 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 055174b2..5c872b5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,6 +2168,7 @@ dependencies = [ "hex", "kinode_process_lib", "nix 0.27.1", + "proc-macro2", "regex", "reqwest", "rpassword", diff --git a/Cargo.toml b/Cargo.toml index 48a62ed5..07ce7e94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ fs-err = "2.11" hex = "0.4" kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib.git", rev = "9ac9e51" } nix = { version = "0.27", features = ["process", "signal", "term"] } +proc-macro2 = "1.0" regex = "1" reqwest = { version = "0.12", features = ["json"] } rpassword = "7" @@ -46,7 +47,8 @@ semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10.8" -syn = { version = "2.0", features = ["full"] } +syn = { version = "2.0", features = ["full", "visit", "extra-traits"] } +#syn = { version = "2.0", features = ["full", "visit"] } thiserror = "1.0" tokio = { version = "1.28", features = [ "macros", diff --git a/src/build/rewrite.rs b/src/build/rewrite.rs index 33c30a12..e43c4508 100644 --- a/src/build/rewrite.rs +++ b/src/build/rewrite.rs @@ -1,10 +1,14 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use color_eyre::{eyre::eyre, Result}; use fs_err as fs; use regex::Regex; -use syn::{__private::ToTokens, parse_str}; +use syn::{ + __private::ToTokens, + parse_str, + visit::{self, Visit}, +}; use toml_edit; use tracing::{debug, instrument}; @@ -43,16 +47,40 @@ impl From for GeneratedProcessesExternal { } } -#[derive(Debug)] +#[derive(Debug, Clone)] struct ArgInfo { name: String, ty: String, } +#[derive(Debug, Clone)] +struct ReturnInfo { + ty: String, +} + +#[derive(Debug, Clone)] +struct FnSignature { + args: Vec, + ret: Option, +} + #[derive(Debug)] -struct SpawnMatch { - args: String, +struct FnInfo { + name: String, + signature: FnSignature, body: String, + dependencies: HashSet, +} + +#[derive(Debug)] +enum SpawnType { + Closure { args: String, body: String }, + FnCall { name: String, args: Vec }, +} + +#[derive(Debug)] +struct SpawnMatch { + spawn_type: SpawnType, imports: Vec, start_pos: usize, end_pos: usize, @@ -64,12 +92,26 @@ enum SpawnParseError { Imports, #[error("Spawn parse failed due to malformed closure: no closing pipe in closure")] NoClosingPipe, - #[error("Spawn parse failed due to malformed closure: no opening brace")] + #[error("Spawn parse failed due to malformed closure: no opening brace `{{`")] NoOpeningBrace, - #[error("Spawn parse failed due to malformed closure: unclosed brace")] + #[error("Spawn parse failed due to malformed closure: no opening paren `(`")] + NoOpeningParen, + #[error("Spawn parse failed due to malformed closure: no opening bracket `[`")] + NoOpeningBracket, + #[error("Spawn parse failed due to malformed closure: unclosed brace `{{`")] UnclosedBrace, - #[error("Spawn parse failed due to malformed closure: unclosed paren")] + #[error("Spawn parse failed due to malformed closure: unclosed paren `(`")] UnclosedParen, + #[error("Spawn parse failed due to malformed closure: unclosed bracket` `[`")] + UnclosedBracket, + #[error("Spawn parse failed: malformed function call")] + MalformedFunctionCall, + #[error("Spawn parse failed: no opening paren for arguments")] + UnclosedArgsParen, + #[error("Spawn parse failed: unclosed spawn paren")] + UnclosedSpawnParen, + #[error("Spawn parse failed: must start with `Spawn!(`")] + InvalidSpawnSyntax, } // TODO: factor out with build::mod.rs::copy_dir() @@ -106,12 +148,12 @@ fn make_args_struct_name(worker_name: &str) -> String { } #[instrument(level = "trace", skip_all)] -fn parse_fn_args(args: &str) -> Result> { +fn parse_fn_signature(args: &str) -> Result { // Parse the argument string as Rust function parameters let fn_item: syn::ItemFn = parse_str(&format!("fn dummy({args}) {{}}"))?; // Extract the parameters from the function signature - let params = fn_item + let args = fn_item .sig .inputs .into_iter() @@ -127,7 +169,15 @@ fn parse_fn_args(args: &str) -> Result> { }) .collect(); - Ok(params) + // Extract return type if present + let ret = match fn_item.sig.output { + syn::ReturnType::Default => None, + syn::ReturnType::Type(_, ty) => Some(ReturnInfo { + ty: ty.into_token_stream().to_string(), + }), + }; + + Ok(FnSignature { args, ret }) } fn generate_args_struct_type(struct_name: &str, args: &[ArgInfo]) -> String { @@ -220,7 +270,135 @@ fn extract_wit_bindgen(content: &str) -> Option { } } -fn parse_spawn_from(input: &str) -> Result<(String, String, usize), SpawnParseError> { +#[instrument(level = "trace", skip_all)] +fn extract_functions(content: &str) -> Result> { + let syntax_tree = syn::parse_file(content)?; + let mut functions = HashMap::new(); + + for item in syntax_tree.items { + if let syn::Item::Fn(func) = item { + let name = func.sig.ident.to_string(); + // Extract both args and return type + let signature = FnSignature { + args: func + .sig + .inputs + .iter() + .filter_map(|arg| { + if let syn::FnArg::Typed(pat_type) = arg { + Some(ArgInfo { + name: pat_type.pat.to_token_stream().to_string(), + ty: pat_type.ty.to_token_stream().to_string(), + }) + } else { + None + } + }) + .collect(), + ret: match &func.sig.output { + syn::ReturnType::Default => None, + syn::ReturnType::Type(_, ty) => Some(ReturnInfo { + ty: ty.into_token_stream().to_string(), + }), + }, + }; + + let mut deps = HashSet::new(); + find_fn_calls(&func.block, &mut deps); + + functions.insert( + name.clone(), + FnInfo { + name, + signature, + body: func.block.to_token_stream().to_string(), + dependencies: deps, + }, + ); + } + } + + Ok(functions) +} + +fn find_fn_calls(block: &syn::Block, deps: &mut HashSet) { + fn inspect_expr(expr: &syn::Expr, deps: &mut HashSet) { + match expr { + syn::Expr::Call(call) => { + // Check direct function call + if let syn::Expr::Path(path) = &*call.func { + if let Some(ident) = path.path.get_ident() { + deps.insert(ident.to_string()); + } + } + // Check arguments recursively + for arg in &call.args { + inspect_expr(arg, deps); + } + } + syn::Expr::Macro(mac) => { + // Convert tokens to string and look for function calls + let tokens = mac.mac.tokens.clone(); + let tokens_str = tokens.to_string(); + + // Split on comma and look at each part + for part in tokens_str.split(',') { + // Look for function call pattern: function_name(args) + if let Some(func_name) = part.trim().split('(').next() { + // Ignore format specifiers and other non-function tokens + if !func_name.contains('"') && !func_name.is_empty() { + deps.insert(func_name.trim().to_string()); + } + } + } + + // Still try to parse as expression for other cases + if let Ok(expr) = syn::parse2::(tokens) { + inspect_expr(&expr, deps); + } + } + syn::Expr::Block(block_expr) => { + for stmt in &block_expr.block.stmts { + inspect_stmt(stmt, deps); + } + } + _ => {} + } + } + + fn inspect_stmt(stmt: &syn::Stmt, deps: &mut HashSet) { + match stmt { + syn::Stmt::Expr(expr, _) => inspect_expr(expr, deps), + syn::Stmt::Local(local) => { + if let Some(init) = &local.init { + inspect_expr(&init.expr, deps); + } + } + syn::Stmt::Macro(mac_stmt) => { + if let Ok(expr) = syn::parse2::(mac_stmt.mac.tokens.clone()) { + inspect_expr(&expr, deps); + } else { + // Handle tokens directly for macro statements too + let tokens_str = mac_stmt.mac.tokens.to_string(); + for part in tokens_str.split(',') { + if let Some(func_name) = part.trim().split('(').next() { + if !func_name.contains('"') && !func_name.is_empty() { + deps.insert(func_name.trim().to_string()); + } + } + } + } + } + _ => {} + } + } + + for stmt in &block.stmts { + inspect_stmt(stmt, deps); + } +} + +fn parse_spawn_closure(input: &str) -> Result<(String, String, usize), SpawnParseError> { // Skip the "Spawn!(|" prefix since we know it's there let input_after_spawn = &input["Spawn!(|".len()..]; @@ -273,19 +451,217 @@ fn parse_spawn_from(input: &str) -> Result<(String, String, usize), SpawnParseEr Ok((args, body, total_consumed)) } +fn parse_spawn_fn_call(input: &str) -> Result<(String, Vec, usize), SpawnParseError> { + // Skip the "Spawn!(" prefix + let input_after_spawn = &input["Spawn!(".len()..]; + + // Find the function name (everything up to first '(' or whitespace) + let name_end = input_after_spawn + .find(|c: char| c == '(' || c.is_whitespace()) + .ok_or(SpawnParseError::MalformedFunctionCall)?; + let name = input_after_spawn[..name_end].trim().to_string(); + + // Find opening paren of args + let args_start = input_after_spawn[name_end..] + .find('(') + .ok_or(SpawnParseError::NoOpeningParen)? + .saturating_add(name_end); + + // Find closing paren while handling nested parens + let mut paren_count = 1; + let mut args_end = None; + let mut closing_spawn_paren = None; + + for (i, c) in input_after_spawn[args_start + 1..].chars().enumerate() { + match c { + '(' => paren_count += 1, + ')' => { + paren_count -= 1; + if paren_count == 0 { + args_end = Some(args_start + 1 + i); + } else if paren_count == -1 { + // This is the closing paren of Spawn!(...) + closing_spawn_paren = Some(args_start + 1 + i); + break; + } + } + _ => {} + } + } + + let args_end = args_end.ok_or(SpawnParseError::UnclosedArgsParen)?; + let closing_spawn_paren = closing_spawn_paren.ok_or(SpawnParseError::UnclosedSpawnParen)?; + + // Parse args list by splitting on commas, handling nested stuff + let args_str = input_after_spawn[args_start + 1..args_end].trim(); + let args = split_args(args_str)?; + + // Return total consumed length including both closing parens + let total_consumed = "Spawn!(".len() + closing_spawn_paren + 1; + + Ok((name, args, total_consumed)) +} + +fn split_args(args: &str) -> Result, SpawnParseError> { + let mut result = Vec::new(); + let mut current = String::new(); + let mut paren_count = 0; + let mut brace_count = 0; + let mut bracket_count = 0; + + for c in args.chars() { + match c { + '(' => paren_count += 1, + ')' => paren_count -= 1, + '{' => brace_count += 1, + '}' => brace_count -= 1, + '[' => bracket_count += 1, + ']' => bracket_count -= 1, + ',' if paren_count == 0 && brace_count == 0 && bracket_count == 0 => { + result.push(current.trim().to_string()); + current = String::new(); + continue; + } + _ => {} + } + current.push(c); + } + + if !current.is_empty() { + result.push(current.trim().to_string()); + } + + if paren_count != 0 { + return Err(SpawnParseError::UnclosedParen); + } + if brace_count != 0 { + return Err(SpawnParseError::UnclosedBrace); + } + if bracket_count != 0 { + return Err(SpawnParseError::UnclosedBracket); + } + + Ok(result) +} + +fn parse_spawn_from(input: &str) -> Result<(SpawnType, usize), SpawnParseError> { + if input.starts_with("Spawn!(|") { + // Existing closure parsing logic + let (args, body, consumed) = parse_spawn_closure(&input)?; + Ok((SpawnType::Closure { args, body }, consumed)) + } else if input.starts_with("Spawn!(") { + // Function call parsing logic + debug!("parsing non-closure `Spawn!(`"); + let (name, args, consumed) = parse_spawn_fn_call(&input)?; + Ok((SpawnType::FnCall { name, args }, consumed)) + } else { + Err(SpawnParseError::InvalidSpawnSyntax) + } +} + +fn add_function_and_deps( + name: &str, + functions: &HashMap, + needed: &mut HashSet, +) { + needed.insert(name.to_string()); + if let Some(info) = functions.get(name) { + for dep in &info.dependencies { + add_function_and_deps(dep, functions, needed); + } + } +} + #[instrument(level = "trace", skip_all)] fn generate_worker_process( process_name: &str, - body: &str, + spawn_match: &SpawnMatch, + functions: &HashMap, imports: &[String], wit_bindgen: &str, args_type: &str, args_destructure: &str, ) -> Result { + let mut needed_fns = HashSet::new(); + + // Get return type if it's a function call + let return_type = match &spawn_match.spawn_type { + SpawnType::FnCall { name, .. } => { + if let Some(fn_info) = functions.get(name) { + fn_info.signature.ret.clone() + } else { + None + } + } + SpawnType::Closure { .. } => None, // Closures don't have return types in our context + }; + + // Get list of functions we need to copy + match &spawn_match.spawn_type { + SpawnType::Closure { body, .. } => { + // Parse body to find function calls + // Add braces back before parsing + let block_str = format!("{{{body}}}"); + let syntax_tree = syn::parse_str::(&block_str)?; + // First find direct function calls in the closure + find_fn_calls(&syntax_tree, &mut needed_fns); + debug!("generate_worker_process find_fn_calls needed_fns {needed_fns:?}"); + debug!("{:?}", functions.keys().collect::>()); + // Then recursively add dependencies for each function found + let direct_deps = needed_fns.clone(); // Clone before recursive traversal + debug!("{direct_deps:?}"); + for name in direct_deps { + add_function_and_deps(&name, functions, &mut needed_fns); + } + } + SpawnType::FnCall { name, .. } => { + // Add the called function and its dependencies + debug!("fncall {name}"); + debug!("{:?}", functions.keys().collect::>()); + add_function_and_deps(name, functions, &mut needed_fns); + } + } + debug!("generate_worker_process found deps: {needed_fns:?}"); + let imports = imports .iter() .map(|i| format!("#[allow(unused_imports)]\nuse {i};\n")) .collect::(); + + // Generate function definitions preserving return types + let function_definitions = needed_fns + .iter() + .filter_map(|name| functions.get(name)) + .map(|info| { + let ret_type = info + .signature + .ret + .as_ref() + .map_or("".to_string(), |r| format!(" -> {}", r.ty)); + format!( + "fn {}({}){}{}", + info.name, + info.signature + .args + .iter() + .map(|arg| format!("{}: {}", arg.name, arg.ty)) + .collect::>() + .join(", "), + ret_type, + info.body, + ) + }) + .collect::(); + + debug!("{function_definitions}"); + + let body = match &spawn_match.spawn_type { + SpawnType::Closure { body, .. } => format!("{body};"), + SpawnType::FnCall { name, args } => { + format!("{name}({});", args.join(", ")) + } + }; + let template = format!( r#"// Generated worker process for {process_name} {imports} @@ -294,6 +670,8 @@ fn generate_worker_process( {args_type} +{function_definitions} + call_init!(init); fn init(our: Address) {{ // Get args from parent @@ -314,14 +692,13 @@ fn find_all_spawns(input: &str) -> Result, SpawnParseError> { let mut search_from = 0; let imports = extract_imports(input)?; - while let Some(spawn_start) = input[search_from..].find("Spawn!(|") { + while let Some(spawn_start) = input[search_from..].find("Spawn!(") { let absolute_start = search_from + spawn_start; - let (args, body, consumed_len) = parse_spawn_from(&input[absolute_start..])?; + let (spawn_type, consumed_len) = parse_spawn_from(&input[absolute_start..])?; results.push(SpawnMatch { - args, - body, + spawn_type, imports: imports.clone(), start_pos: absolute_start, end_pos: absolute_start + consumed_len, @@ -343,21 +720,36 @@ fn rewrite_rust_file( ) -> Result { let spawn_matches = find_all_spawns(content)?; let mut new_content = content.to_string(); + let functions = extract_functions(&content)?; + debug!("got functions in {process_name}: {:#?}", functions); // Process spawns in reverse order to not invalidate positions for (i, spawn_match) in spawn_matches.iter().enumerate().rev() { let worker_name = format!("{process_name}-worker-{i}"); let wasm_name = format!("{worker_name}.wasm"); - let args_name = make_args_struct_name(&worker_name); - let parsed_args = parse_fn_args(&spawn_match.args)?; - let args_type = generate_args_struct_type(&args_name, &parsed_args); - let args_instance = generate_args_struct_instance(&args_name, &parsed_args); - let args_destructure = generate_args_struct_destructure(&args_name, &parsed_args); + let (args_name, parsed_signature) = match &spawn_match.spawn_type { + SpawnType::Closure { args, .. } => { + let args_name = make_args_struct_name(&worker_name); + let parsed_signature = parse_fn_signature(args)?; + (args_name, parsed_signature) + } + SpawnType::FnCall { name, args } => { + let fn_info = functions + .get(name) + .ok_or_else(|| eyre!("Function {name} not found in parent"))?; + // For function calls, use the function's argument names but + // with the values supplied in the Spawn!() call + let args_name = make_args_struct_name(&worker_name); + (args_name, fn_info.signature.clone()) + } + }; + + let args_type = generate_args_struct_type(&args_name, &parsed_signature.args); + let args_destructure = generate_args_struct_destructure(&args_name, &parsed_signature.args); // Generate worker process let wit_bindgen = extract_wit_bindgen(content).unwrap_or_else(|| { - // Fallback to default if not found r#"wit_bindgen::generate!({ path: "target/wit", world: "process-v0", @@ -367,7 +759,8 @@ fn rewrite_rust_file( let worker_code = generate_worker_process( process_name, - &spawn_match.body, + spawn_match, + &functions, &spawn_match.imports, &wit_bindgen, &args_type, @@ -381,7 +774,31 @@ fn rewrite_rust_file( .or_default() .insert(worker_name.clone(), (wasm_name, worker_code)); - // Create replacement spawn code + // Create replacement spawn code with appropriate args instantiation + let args_instance = match &spawn_match.spawn_type { + SpawnType::Closure { args, .. } => { + // For closures, use the argument names directly + generate_args_struct_instance(&args_name, &parsed_signature.args) + } + SpawnType::FnCall { args, .. } => { + // For function calls, use the supplied argument values + let fields = parsed_signature + .args + .iter() + .zip(args.iter()) + .map(|(arg, value)| format!(" {}: {},", arg.name, value)) + .collect::>() + .join("\n"); + + format!( + r#"let args = {args_name} {{ +{fields} + }};"# + ) + } + }; + + // Create the replacement code let replacement = format!( r#"{{ use kinode_process_lib::{{spawn, OnExit, Request}}; From cfc7b506de01ca1d46b10fe1bb2d79e159c25838 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 22 Nov 2024 20:38:17 -0800 Subject: [PATCH 16/17] build: change default to NOT rewrite --- src/build/mod.rs | 20 ++++++++++---------- src/build_start_package/mod.rs | 4 ++-- src/main.rs | 18 +++++++++--------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 5330cc5e..c24c3c64 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1158,7 +1158,7 @@ async fn fetch_dependencies( default_world: Option<&str>, include: &HashSet, exclude: &HashSet, - no_rewrite: bool, + rewrite: bool, force: bool, verbose: bool, ) -> Result<()> { @@ -1175,7 +1175,7 @@ async fn fetch_dependencies( default_world, vec![], // TODO: what about deps-of-deps? vec![], - no_rewrite, + rewrite, false, force, verbose, @@ -1212,7 +1212,7 @@ async fn fetch_dependencies( default_world, local_dep_deps, vec![], - no_rewrite, + rewrite, false, force, verbose, @@ -1528,7 +1528,7 @@ async fn compile_package( add_paths_to_api: &Vec, include: &HashSet, exclude: &HashSet, - no_rewrite: bool, + rewrite: bool, force: bool, verbose: bool, ignore_deps: bool, // for internal use; may cause problems when adding recursive deps @@ -1551,7 +1551,7 @@ async fn compile_package( default_world, include, exclude, - no_rewrite, + rewrite, force, verbose, ) @@ -1659,7 +1659,7 @@ pub async fn execute( default_world: Option<&str>, local_dependencies: Vec, add_paths_to_api: Vec, - no_rewrite: bool, + rewrite: bool, reproducible: bool, force: bool, verbose: bool, @@ -1744,9 +1744,9 @@ pub async fn execute( check_process_lib_version(&package_dir.join("Cargo.toml"))?; // live_dir is the "dir that is being built" or is "live"; - // if `no_rewrite`, that is just `package_dir`; + // if `!rewrite`, that is just `package_dir`; // else, it is the modified copy that is in `target/rewrite/` - let live_dir = if no_rewrite { + let live_dir = if !rewrite { PathBuf::from(package_dir) } else { copy_and_rewrite_package(package_dir)? @@ -1777,7 +1777,7 @@ pub async fn execute( &add_paths_to_api, &include, &exclude, - no_rewrite, + rewrite, force, verbose, ignore_deps, @@ -1785,7 +1785,7 @@ pub async fn execute( .await?; } - if !no_rewrite { + if rewrite { if package_dir.join("pkg").exists() { fs::remove_dir_all(package_dir.join("pkg"))?; } diff --git a/src/build_start_package/mod.rs b/src/build_start_package/mod.rs index 36eec647..d3707196 100644 --- a/src/build_start_package/mod.rs +++ b/src/build_start_package/mod.rs @@ -21,7 +21,7 @@ pub async fn execute( default_world: Option<&str>, local_dependencies: Vec, add_paths_to_api: Vec, - no_rewrite: bool, + rewrite: bool, reproducible: bool, force: bool, verbose: bool, @@ -39,7 +39,7 @@ pub async fn execute( default_world, local_dependencies, add_paths_to_api, - no_rewrite, + rewrite, reproducible, force, verbose, diff --git a/src/main.rs b/src/main.rs index c5d1892f..f148e74a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -222,7 +222,7 @@ async fn execute( .unwrap_or_default() .map(|s| PathBuf::from(s)) .collect(); - let no_rewrite = matches.get_one::("NO_REWRITE").unwrap(); + let rewrite = matches.get_one::("REWRITE").unwrap(); let reproducible = matches.get_one::("REPRODUCIBLE").unwrap(); let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); @@ -240,7 +240,7 @@ async fn execute( default_world.map(|w| w.as_str()), local_dependencies, add_paths_to_api, - *no_rewrite, + *rewrite, *reproducible, *force, *verbose, @@ -285,7 +285,7 @@ async fn execute( .unwrap_or_default() .map(|s| PathBuf::from(s)) .collect(); - let no_rewrite = matches.get_one::("NO_REWRITE").unwrap(); + let rewrite = matches.get_one::("REWRITE").unwrap(); let reproducible = matches.get_one::("REPRODUCIBLE").unwrap(); let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); @@ -303,7 +303,7 @@ async fn execute( default_world.map(|w| w.as_str()), local_dependencies, add_paths_to_api, - *no_rewrite, + *rewrite, *reproducible, *force, *verbose, @@ -737,10 +737,10 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .long("add-to-api") .help("Path to file to add to api.zip (can specify multiple times)") ) - .arg(Arg::new("NO_REWRITE") + .arg(Arg::new("REWRITE") .action(ArgAction::SetTrue) - .long("no-rewrite") - .help("Don't rewrite the package (disables `Spawn!()`) [default: rewrite]") + .long("rewrite") + .help("Rewrite the package (disables `Spawn!()`) [default: don't rewrite]") .required(false) ) .arg(Arg::new("REPRODUCIBLE") @@ -844,10 +844,10 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Pass these comma-delimited feature flags to Rust cargo builds") .required(false) ) - .arg(Arg::new("NO_REWRITE") + .arg(Arg::new("REWRITE") .action(ArgAction::SetTrue) .long("no-rewrite") - .help("Don't rewrite the package (disables `Spawn!()`) [default: rewrite]") + .help("Rewrite the package (disables `Spawn!()`) [default: don't rewrite]") .required(false) ) .arg(Arg::new("REPRODUCIBLE") From 2ca648054431da62daab22c88d663aaa8a254898 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Mon, 9 Dec 2024 16:32:00 -0500 Subject: [PATCH 17/17] add wit v1 as build option --- src/build/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index c24c3c64..ca04028a 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -42,6 +42,8 @@ const KINODE_WIT_0_7_0_URL: &str = "https://raw.githubusercontent.com/kinode-dao/kinode-wit/aa2c8b11c9171b949d1991c32f58591c0e881f85/kinode.wit"; const KINODE_WIT_0_8_0_URL: &str = "https://raw.githubusercontent.com/kinode-dao/kinode-wit/v0.8/kinode.wit"; +const KINODE_WIT_1_0_0_URL: &str = + "https://raw.githubusercontent.com/kinode-dao/kinode-wit/1.0/kinode.wit"; const WASI_VERSION: &str = "19.0.1"; // TODO: un-hardcode const DEFAULT_WORLD_0_7_0: &str = "process"; const DEFAULT_WORLD_0_8_0: &str = "process-v0"; @@ -1074,7 +1076,8 @@ async fn build_wit_dir( } let wit_url = match wit_version { None => KINODE_WIT_0_7_0_URL, - Some(0) | _ => KINODE_WIT_0_8_0_URL, + Some(0) => KINODE_WIT_0_8_0_URL, + Some(1) | _ => KINODE_WIT_1_0_0_URL, }; download_file(wit_url, &wit_dir.join("kinode.wit")).await?; for (file_name, contents) in apis {