From 96dbf39be1ca4aaa9c037be879902085226f3c25 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 3 Apr 2025 17:20:08 -0700 Subject: [PATCH 1/8] add hyper-bindgen from commit: https://github.com/jaxs-ribs/hyper-bindgen/commit/858481b301c534acf9708cf849bfcdbab5c75dcf --- src/build/caller_utils_generator.rs | 786 +++++++++++++++++++++++ src/build/wit_generator.rs | 945 ++++++++++++++++++++++++++++ 2 files changed, 1731 insertions(+) create mode 100644 src/build/caller_utils_generator.rs create mode 100644 src/build/wit_generator.rs diff --git a/src/build/caller_utils_generator.rs b/src/build/caller_utils_generator.rs new file mode 100644 index 00000000..8de9c0f9 --- /dev/null +++ b/src/build/caller_utils_generator.rs @@ -0,0 +1,786 @@ +use anyhow::{Context, Result, bail}; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use toml::Value; +use walkdir::WalkDir; + +// Convert kebab-case to snake_case +pub fn to_snake_case(s: &str) -> String { + s.replace('-', "_") +} + +// Convert kebab-case to PascalCase +pub fn to_pascal_case(s: &str) -> String { + let parts = s.split('-'); + let mut result = String::new(); + + for part in parts { + if !part.is_empty() { + let mut chars = part.chars(); + if let Some(first_char) = chars.next() { + result.push(first_char.to_uppercase().next().unwrap()); + result.extend(chars); + } + } + } + + result +} + +// Find the world name in the world WIT file, prioritizing types-prefixed worlds +fn find_world_name(api_dir: &Path) -> Result { + let mut regular_world_name = None; + let mut types_world_name = None; + + // Look for world definition files + for entry in WalkDir::new(api_dir) + .max_depth(1) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { + if let Ok(content) = fs::read_to_string(path) { + if content.contains("world ") { + println!("Analyzing world definition file: {}", path.display()); + + // Extract the world name + let lines: Vec<&str> = content.lines().collect(); + + if let Some(world_line) = lines.iter().find(|line| line.trim().starts_with("world ")) { + println!("World line: {}", world_line); + + if let Some(world_name) = world_line.trim().split_whitespace().nth(1) { + let clean_name = world_name.trim_end_matches(" {"); + println!("Extracted world name: {}", clean_name); + + // Check if this is a types-prefixed world + if clean_name.starts_with("types-") { + types_world_name = Some(clean_name.to_string()); + println!("Found types world: {}", clean_name); + } else { + regular_world_name = Some(clean_name.to_string()); + println!("Found regular world: {}", clean_name); + } + } + } + } + } + } + } + + // Prioritize types-prefixed world if found + if let Some(types_name) = types_world_name { + return Ok(types_name); + } + + // If no types-prefixed world found, check if we have a regular world + if let Some(regular_name) = regular_world_name { + // Check if there's a corresponding types-prefixed world file + let types_name = format!("types-{}", regular_name); + let types_file = api_dir.join(format!("{}.wit", types_name)); + + if types_file.exists() { + println!("Found types world from file: {}", types_name); + return Ok(types_name); + } + + // Fall back to regular world but print a warning + println!("Warning: No types- world found, using regular world: {}", regular_name); + return Ok(regular_name); + } + + // If no world name is found, we should fail + bail!("No world name found in any WIT file. Cannot generate caller-utils without a world name.") +} + +// Convert WIT type to Rust type - IMPROVED with more Rust primitives +fn wit_type_to_rust(wit_type: &str) -> String { + match wit_type { + // Integer types + "s8" => "i8".to_string(), + "u8" => "u8".to_string(), + "s16" => "i16".to_string(), + "u16" => "u16".to_string(), + "s32" => "i32".to_string(), + "u32" => "u32".to_string(), + "s64" => "i64".to_string(), + "u64" => "u64".to_string(), + // Size types + "usize" => "usize".to_string(), + "isize" => "isize".to_string(), + // Floating point types + "f32" => "f32".to_string(), + "f64" => "f64".to_string(), + // Other primitive types + "string" => "String".to_string(), + "str" => "&str".to_string(), + "char" => "char".to_string(), + "bool" => "bool".to_string(), + "unit" => "()".to_string(), + // Special types + "address" => "WitAddress".to_string(), + // Common primitives that might be written differently in WIT + "i8" => "i8".to_string(), + "i16" => "i16".to_string(), + "i32" => "i32".to_string(), + "i64" => "i64".to_string(), + // Collection types with generics + t if t.starts_with("list<") => { + let inner_type = &t[5..t.len() - 1]; + format!("Vec<{}>", wit_type_to_rust(inner_type)) + }, + t if t.starts_with("option<") => { + let inner_type = &t[7..t.len() - 1]; + format!("Option<{}>", wit_type_to_rust(inner_type)) + }, + t if t.starts_with("result<") => { + let inner_part = &t[7..t.len() - 1]; + if let Some(comma_pos) = inner_part.find(',') { + let ok_type = &inner_part[..comma_pos].trim(); + let err_type = &inner_part[comma_pos + 1..].trim(); + format!("Result<{}, {}>", wit_type_to_rust(ok_type), wit_type_to_rust(err_type)) + } else { + format!("Result<{}, ()>", wit_type_to_rust(inner_part)) + } + }, + t if t.starts_with("tuple<") => { + let inner_types = &t[6..t.len() - 1]; + let rust_types: Vec = inner_types + .split(", ") + .map(|t| wit_type_to_rust(t)) + .collect(); + format!("({})", rust_types.join(", ")) + }, + // Handle map type if present + t if t.starts_with("map<") => { + let inner_part = &t[4..t.len() - 1]; + if let Some(comma_pos) = inner_part.find(',') { + let key_type = &inner_part[..comma_pos].trim(); + let value_type = &inner_part[comma_pos + 1..].trim(); + format!("HashMap<{}, {}>", wit_type_to_rust(key_type), wit_type_to_rust(value_type)) + } else { + // Fallback for malformed map type + format!("HashMap", wit_type_to_rust(inner_part)) + } + }, + // Custom types (in kebab-case) need to be converted to PascalCase + _ => to_pascal_case(wit_type).to_string(), + } +} + +// Generate default value for Rust type - IMPROVED with additional types +fn generate_default_value(rust_type: &str) -> String { + match rust_type { + // Integer types + "i8" | "u8" | "i16" | "u16" | "i32" | "u32" | "i64" | "u64" | "isize" | "usize" => "0".to_string(), + // Floating point types + "f32" | "f64" => "0.0".to_string(), + // String types + "String" => "String::new()".to_string(), + "&str" => "\"\"".to_string(), + // Other primitive types + "bool" => "false".to_string(), + "char" => "'\\0'".to_string(), + "()" => "()".to_string(), + // Collection types + t if t.starts_with("Vec<") => "Vec::new()".to_string(), + t if t.starts_with("Option<") => "None".to_string(), + t if t.starts_with("Result<") => { + // For Result, default to Ok with the default value of the success type + if let Some(success_type_end) = t.find(',') { + let success_type = &t[7..success_type_end]; + format!("Ok({})", generate_default_value(success_type)) + } else { + "Ok(())".to_string() + } + }, + t if t.starts_with("HashMap<") => "HashMap::new()".to_string(), + t if t.starts_with("(") => { + // Generate default tuple with default values for each element + let inner_part = t.trim_start_matches('(').trim_end_matches(')'); + let parts: Vec<_> = inner_part.split(", ").collect(); + let default_values: Vec<_> = parts.iter() + .map(|part| generate_default_value(part)) + .collect(); + format!("({})", default_values.join(", ")) + }, + // For custom types, assume they implement Default + _ => format!("{}::default()", rust_type), + } +} + +// Structure to represent a field in a WIT signature struct +struct SignatureField { + name: String, + wit_type: String, +} + +// Structure to represent a WIT signature struct +struct SignatureStruct { + function_name: String, + attr_type: String, + fields: Vec, +} + +// Find all interface imports in the world WIT file +fn find_interfaces_in_world(api_dir: &Path) -> Result> { + let mut interfaces = Vec::new(); + + // Find world definition files + for entry in WalkDir::new(api_dir) + .max_depth(1) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { + if let Ok(content) = fs::read_to_string(path) { + if content.contains("world ") { + println!("Analyzing world definition file: {}", path.display()); + + // Extract import statements + for line in content.lines() { + let line = line.trim(); + if line.starts_with("import ") && line.ends_with(";") { + let interface = line + .trim_start_matches("import ") + .trim_end_matches(";") + .trim(); + + interfaces.push(interface.to_string()); + println!(" Found interface import: {}", interface); + } + } + } + } + } + } + + Ok(interfaces) +} + +// Parse WIT file to extract function signatures and type definitions +fn parse_wit_file(file_path: &Path) -> Result<(Vec, Vec)> { + println!("Parsing WIT file: {}", file_path.display()); + + let content = fs::read_to_string(file_path) + .with_context(|| format!("Failed to read WIT file: {}", file_path.display()))?; + + let mut signatures = Vec::new(); + let mut type_names = Vec::new(); + + // Simple parser for WIT files to extract record definitions and types + let lines: Vec<_> = content.lines().collect(); + let mut i = 0; + + while i < lines.len() { + let line = lines[i].trim(); + + // Look for record definitions that aren't signature structs + if line.starts_with("record ") && !line.contains("-signature-") { + let record_name = line.trim_start_matches("record ").trim_end_matches(" {").trim(); + println!(" Found type: record {}", record_name); + type_names.push(record_name.to_string()); + } + // Look for variant definitions (enums) + else if line.starts_with("variant ") { + let variant_name = line.trim_start_matches("variant ").trim_end_matches(" {").trim(); + println!(" Found type: variant {}", variant_name); + type_names.push(variant_name.to_string()); + } + // Look for signature record definitions + else if line.starts_with("record ") && line.contains("-signature-") { + let record_name = line.trim_start_matches("record ").trim_end_matches(" {").trim(); + println!(" Found record: {}", record_name); + + // Extract function name and attribute type + let parts: Vec<_> = record_name.split("-signature-").collect(); + if parts.len() != 2 { + println!(" Unexpected record name format"); + i += 1; + continue; + } + + let function_name = parts[0].to_string(); + let attr_type = parts[1].to_string(); + + // Parse fields + let mut fields = Vec::new(); + i += 1; + + while i < lines.len() && !lines[i].trim().starts_with("}") { + let field_line = lines[i].trim(); + + // Skip comments and empty lines + if field_line.starts_with("//") || field_line.is_empty() { + i += 1; + continue; + } + + // Parse field definition + let field_parts: Vec<_> = field_line.split(':').collect(); + if field_parts.len() == 2 { + let field_name = field_parts[0].trim().to_string(); + let field_type = field_parts[1].trim().trim_end_matches(',').to_string(); + + println!(" Field: {} -> {}", field_name, field_type); + fields.push(SignatureField { + name: field_name, + wit_type: field_type, + }); + } + + i += 1; + } + + signatures.push(SignatureStruct { + function_name, + attr_type, + fields, + }); + } + + i += 1; + } + + println!("Extracted {} signature structs and {} type definitions from {}", + signatures.len(), type_names.len(), file_path.display()); + Ok((signatures, type_names)) +} + +// Generate a Rust async function from a signature struct +fn generate_async_function(signature: &SignatureStruct) -> String { + // Convert function name from kebab-case to snake_case + let snake_function_name = to_snake_case(&signature.function_name); + + // Get pascal case version for the JSON request format + let pascal_function_name = to_pascal_case(&signature.function_name); + + // Function full name with attribute type + let full_function_name = format!("{}_{}_rpc", snake_function_name, signature.attr_type); + + // Extract parameters and return type + let mut params = Vec::new(); + let mut param_names = Vec::new(); + let mut return_type = "()".to_string(); + let mut target_param = ""; + + for field in &signature.fields { + let field_name_snake = to_snake_case(&field.name); + let rust_type = wit_type_to_rust(&field.wit_type); + + if field.name == "target" { + if field.wit_type == "string" { + target_param = "&str"; + } else { + // Use hyperware_process_lib::Address instead of WitAddress + target_param = "&Address"; + } + } else if field.name == "returning" { + return_type = rust_type; + } else { + params.push(format!("{}: {}", field_name_snake, rust_type)); + param_names.push(field_name_snake); + } + } + + // First parameter is always target + let all_params = if target_param.is_empty() { + params.join(", ") + } else { + format!("target: {}{}", target_param, if params.is_empty() { "" } else { ", " }) + ¶ms.join(", ") + }; + + // Wrap the return type in SendResult + let wrapped_return_type = format!("SendResult<{}>", return_type); + + // For HTTP endpoints, generate commented-out implementation + if signature.attr_type == "http" { + let default_value = generate_default_value(&return_type); + + // Add underscore prefix to all parameters for HTTP stubs + let all_params_with_underscore = if target_param.is_empty() { + params.iter() + .map(|param| { + let parts: Vec<&str> = param.split(':').collect(); + if parts.len() == 2 { + format!("_{}: {}", parts[0], parts[1]) + } else { + format!("_{}", param) + } + }) + .collect::>() + .join(", ") + } else { + let target_with_underscore = format!("_target: {}", target_param); + if params.is_empty() { + target_with_underscore + } else { + let params_with_underscore = params.iter() + .map(|param| { + let parts: Vec<&str> = param.split(':').collect(); + if parts.len() == 2 { + format!("_{}: {}", parts[0], parts[1]) + } else { + format!("_{}", param) + } + }) + .collect::>() + .join(", "); + format!("{}, {}", target_with_underscore, params_with_underscore) + } + }; + + return format!( + "/// Generated stub for `{}` {} RPC call\n/// HTTP endpoint - uncomment to implement\n// pub async fn {}({}) -> {} {{\n// // TODO: Implement HTTP endpoint\n// SendResult::Success({})\n// }}", + signature.function_name, + signature.attr_type, + full_function_name, + all_params_with_underscore, + wrapped_return_type, + default_value + ); + } + + // Format JSON parameters correctly + let json_params = if param_names.is_empty() { + // No parameters case + format!("json!({{\"{}\" : {{}}}})", pascal_function_name) + } else if param_names.len() == 1 { + // Single parameter case + format!("json!({{\"{}\": {}}})", pascal_function_name, param_names[0]) + } else { + // Multiple parameters case - use tuple format + format!("json!({{\"{}\": ({})}})", + pascal_function_name, + param_names.join(", ")) + }; + + // Generate function with implementation using send + format!( + "/// Generated stub for `{}` {} RPC call\npub async fn {}({}) -> {} {{\n let request = {};\n send::<{}>(&request, target, 30).await\n}}", + signature.function_name, + signature.attr_type, + full_function_name, + all_params, + wrapped_return_type, + json_params, + return_type + ) +} + +// Create the caller-utils crate with a single lib.rs file +fn create_caller_utils_crate(api_dir: &Path, base_dir: &Path) -> Result<()> { + // Path to the new crate + let caller_utils_dir = base_dir.join("caller-utils"); + println!("Creating caller-utils crate at {}", caller_utils_dir.display()); + + // Create directories + fs::create_dir_all(&caller_utils_dir)?; + fs::create_dir_all(caller_utils_dir.join("src"))?; + println!("Created project directory structure"); + + // Create Cargo.toml with updated dependencies + let cargo_toml = r#"[package] +name = "caller-utils" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1.0" +hyperware_process_lib = { version = "1.0.4", features = ["logging"] } +process_macros = "0.1.0" +futures-util = "0.3" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +hyperware_app_common = { git = "https://github.com/hyperware-ai/hyperprocess-macro" } +once_cell = "1.20.2" +futures = "0.3" +uuid = { version = "1.0" } +wit-bindgen = "0.41.0" + +[lib] +crate-type = ["cdylib", "lib"] +"#; + + fs::write(caller_utils_dir.join("Cargo.toml"), cargo_toml) + .with_context(|| "Failed to write caller-utils Cargo.toml")?; + + println!("Created Cargo.toml for caller-utils"); + + // Get the world name (preferably the types- version) + let world_name = find_world_name(api_dir)?; + println!("Using world name for code generation: {}", world_name); + + // Get all interfaces from the world file + let interface_imports = find_interfaces_in_world(api_dir)?; + + // Store all types from each interface + let mut interface_types: HashMap> = HashMap::new(); + + // Find all WIT files in the api directory to generate stubs + let mut wit_files = Vec::new(); + for entry in WalkDir::new(api_dir) + .max_depth(1) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { + // Exclude world definition files + if let Ok(content) = fs::read_to_string(path) { + if !content.contains("world ") { + wit_files.push(path.to_path_buf()); + } + } + } + } + + println!("Found {} WIT interface files", wit_files.len()); + + // Generate content for each module and collect types + let mut module_contents = HashMap::::new(); + + for wit_file in &wit_files { + // Extract the interface name from the file name + let interface_name = wit_file.file_stem().unwrap().to_string_lossy(); + let snake_interface_name = to_snake_case(&interface_name); + + println!("Processing interface: {} -> {}", interface_name, snake_interface_name); + + // Parse the WIT file to extract signature structs and types + match parse_wit_file(wit_file) { + Ok((signatures, types)) => { + // Store types for this interface + interface_types.insert(interface_name.to_string(), types); + + if signatures.is_empty() { + println!("No signatures found in {}", wit_file.display()); + continue; + } + + // Generate module content + let mut mod_content = String::new(); + + // Add function implementations + for signature in &signatures { + let function_impl = generate_async_function(signature); + mod_content.push_str(&function_impl); + mod_content.push_str("\n\n"); + } + + // Store the module content + module_contents.insert(snake_interface_name, mod_content); + + println!("Generated module content with {} function stubs", signatures.len()); + }, + Err(e) => { + println!("Error parsing WIT file {}: {}", wit_file.display(), e); + } + } + } + + // Create import statements for each interface using "hyperware::process::{interface_name}::*" + // Use a HashSet to track which interfaces we've already processed to avoid duplicates + let mut processed_interfaces = std::collections::HashSet::new(); + let mut interface_use_statements = Vec::new(); + + for interface_name in &interface_imports { + // Convert to snake case for module name + let snake_interface_name = to_snake_case(interface_name); + + // Only add the import if we haven't processed this interface yet + if processed_interfaces.insert(snake_interface_name.clone()) { + // Create wildcard import for this interface + interface_use_statements.push( + format!("pub use crate::hyperware::process::{}::*;", snake_interface_name) + ); + } + } + + // Create single lib.rs with all modules inline + let mut lib_rs = String::new(); + + // Updated wit_bindgen usage with explicit world name - FIXED: Removed unused imports + lib_rs.push_str("wit_bindgen::generate!({\n"); + lib_rs.push_str(" path: \"target/wit\",\n"); + lib_rs.push_str(&format!(" world: \"{}\",\n", world_name)); + lib_rs.push_str(" generate_unused_types: true,\n"); + lib_rs.push_str(" additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto],\n"); + lib_rs.push_str("});\n\n"); + + lib_rs.push_str("/// Generated caller utilities for RPC function stubs\n\n"); + + // Add global imports + lib_rs.push_str("pub use hyperware_app_common::SendResult;\n"); + lib_rs.push_str("pub use hyperware_app_common::send;\n"); + lib_rs.push_str("use hyperware_process_lib::Address;\n"); + lib_rs.push_str("use serde_json::json;\n\n"); + + // Add interface use statements + if !interface_use_statements.is_empty() { + lib_rs.push_str("// Import types from each interface\n"); + for use_stmt in interface_use_statements { + lib_rs.push_str(&format!("{}\n", use_stmt)); + } + lib_rs.push_str("\n"); + } + + // Add all modules with their content + for (module_name, module_content) in module_contents { + lib_rs.push_str(&format!("/// Generated RPC stubs for the {} interface\n", module_name)); + lib_rs.push_str(&format!("pub mod {} {{\n", module_name)); + lib_rs.push_str(" use crate::*;\n\n"); + lib_rs.push_str(&format!(" {}\n", module_content.replace("\n", "\n "))); + lib_rs.push_str("}\n\n"); + } + + // Write lib.rs + let lib_rs_path = caller_utils_dir.join("src").join("lib.rs"); + println!("Writing lib.rs to {}", lib_rs_path.display()); + + fs::write(&lib_rs_path, lib_rs) + .with_context(|| format!("Failed to write lib.rs: {}", lib_rs_path.display()))?; + + println!("Created single lib.rs file with all modules inline"); + + // Create target/wit directory and copy all WIT files + let target_wit_dir = caller_utils_dir.join("target").join("wit"); + println!("Creating directory: {}", target_wit_dir.display()); + + // Remove the directory if it exists to ensure clean state + if target_wit_dir.exists() { + println!("Removing existing target/wit directory"); + fs::remove_dir_all(&target_wit_dir)?; + } + + fs::create_dir_all(&target_wit_dir)?; + + // Copy all WIT files to target/wit + for entry in WalkDir::new(api_dir) + .max_depth(1) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { + let file_name = path.file_name().unwrap(); + let target_path = target_wit_dir.join(file_name); + fs::copy(path, &target_path) + .with_context(|| format!("Failed to copy {} to {}", path.display(), target_path.display()))?; + println!("Copied {} to target/wit directory", file_name.to_string_lossy()); + } + } + + Ok(()) +} + +// Update workspace Cargo.toml to include the caller-utils crate +fn update_workspace_cargo_toml(base_dir: &Path) -> Result<()> { + let workspace_cargo_toml = base_dir.join("Cargo.toml"); + println!("Updating workspace Cargo.toml at {}", workspace_cargo_toml.display()); + + if !workspace_cargo_toml.exists() { + println!("Workspace Cargo.toml not found at {}", workspace_cargo_toml.display()); + return Ok(()); + } + + let content = fs::read_to_string(&workspace_cargo_toml) + .with_context(|| format!("Failed to read workspace Cargo.toml: {}", workspace_cargo_toml.display()))?; + + // Parse the TOML content + let mut parsed_toml: Value = content.parse() + .with_context(|| "Failed to parse workspace Cargo.toml")?; + + // Check if there's a workspace section + if let Some(workspace) = parsed_toml.get_mut("workspace") { + if let Some(members) = workspace.get_mut("members") { + if let Some(members_array) = members.as_array_mut() { + // Check if caller-utils is already in the members list + let caller_utils_exists = members_array.iter().any(|m| { + m.as_str().map_or(false, |s| s == "caller-utils") + }); + + if !caller_utils_exists { + println!("Adding caller-utils to workspace members"); + members_array.push(Value::String("caller-utils".to_string())); + + // Write back the updated TOML + let updated_content = toml::to_string_pretty(&parsed_toml) + .with_context(|| "Failed to serialize updated workspace Cargo.toml")?; + + fs::write(&workspace_cargo_toml, updated_content) + .with_context(|| format!("Failed to write updated workspace Cargo.toml: {}", workspace_cargo_toml.display()))?; + + println!("Successfully updated workspace Cargo.toml"); + } else { + println!("caller-utils is already in workspace members"); + } + } + } + } + + Ok(()) +} + +// Add caller-utils as a dependency to hyperware:process crates +fn add_caller_utils_to_projects(projects: &[PathBuf]) -> Result<()> { + for project_path in projects { + let cargo_toml_path = project_path.join("Cargo.toml"); + println!("Adding caller-utils dependency to {}", cargo_toml_path.display()); + + let content = fs::read_to_string(&cargo_toml_path) + .with_context(|| format!("Failed to read project Cargo.toml: {}", cargo_toml_path.display()))?; + + let mut parsed_toml: Value = content.parse() + .with_context(|| format!("Failed to parse project Cargo.toml: {}", cargo_toml_path.display()))?; + + // Add caller-utils to dependencies if not already present + if let Some(dependencies) = parsed_toml.get_mut("dependencies") { + if let Some(deps_table) = dependencies.as_table_mut() { + if !deps_table.contains_key("caller-utils") { + deps_table.insert( + "caller-utils".to_string(), + Value::Table({ + let mut t = toml::map::Map::new(); + t.insert("path".to_string(), Value::String("../caller-utils".to_string())); + t + }) + ); + + // Write back the updated TOML + let updated_content = toml::to_string_pretty(&parsed_toml) + .with_context(|| format!("Failed to serialize updated project Cargo.toml: {}", cargo_toml_path.display()))?; + + fs::write(&cargo_toml_path, updated_content) + .with_context(|| format!("Failed to write updated project Cargo.toml: {}", cargo_toml_path.display()))?; + + println!("Successfully added caller-utils dependency"); + } else { + println!("caller-utils dependency already exists"); + } + } + } + } + + Ok(()) +} + +// Create caller-utils crate and integrate with the workspace +pub fn create_caller_utils(base_dir: &Path, api_dir: &Path, projects: &[PathBuf]) -> Result<()> { + // Step 1: Create the caller-utils crate + create_caller_utils_crate(api_dir, base_dir)?; + + // Step 2: Update workspace Cargo.toml + update_workspace_cargo_toml(base_dir)?; + + // Step 3: Add caller-utils dependency to each hyperware:process project + add_caller_utils_to_projects(projects)?; + + Ok(()) +} \ No newline at end of file diff --git a/src/build/wit_generator.rs b/src/build/wit_generator.rs new file mode 100644 index 00000000..e5ad6949 --- /dev/null +++ b/src/build/wit_generator.rs @@ -0,0 +1,945 @@ +use anyhow::{Context, Result}; +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use syn::{self, Attribute, ImplItem, Item, Type}; +use walkdir::WalkDir; +use toml::Value; + +// Helper functions for naming conventions +fn to_kebab_case(s: &str) -> String { + // First, handle the case where the input has underscores + if s.contains('_') { + return s.replace('_', "-"); + } + + let mut result = String::with_capacity(s.len() + 5); // Extra capacity for hyphens + let chars: Vec = s.chars().collect(); + + for (i, &c) in chars.iter().enumerate() { + if c.is_uppercase() { + // Add hyphen if: + // 1. Not the first character + // 2. Previous character is lowercase + // 3. Or next character is lowercase (to handle acronyms like HTML) + if i > 0 && + (chars[i-1].is_lowercase() || + (i < chars.len() - 1 && chars[i+1].is_lowercase())) + { + result.push('-'); + } + result.push(c.to_lowercase().next().unwrap()); + } else { + result.push(c); + } + } + + result +} + +// Validates a name doesn't contain numbers or "stream" +fn validate_name(name: &str, kind: &str) -> Result<()> { + // Check for numbers + if name.chars().any(|c| c.is_digit(10)) { + anyhow::bail!("Error: {} name '{}' contains numbers, which is not allowed", kind, name); + } + + // Check for "stream" + if name.to_lowercase().contains("stream") { + anyhow::bail!("Error: {} name '{}' contains 'stream', which is not allowed", kind, name); + } + + Ok(()) +} + +// Remove "State" suffix from a name +fn remove_state_suffix(name: &str) -> String { + if name.ends_with("State") { + let len = name.len(); + return name[0..len-5].to_string(); + } + name.to_string() +} + +// Extract wit_world from the #[hyperprocess] attribute using the format in the debug representation +fn extract_wit_world(attrs: &[Attribute]) -> Result { + for attr in attrs { + if attr.path().is_ident("hyperprocess") { + // Convert attribute to string representation + let attr_str = format!("{:?}", attr); + println!("Attribute string: {}", attr_str); + + // Look for wit_world in the attribute string + if let Some(pos) = attr_str.find("wit_world") { + println!("Found wit_world at position {}", pos); + + // Find the literal value after wit_world by looking for lit: "value" + let lit_pattern = "lit: \""; + if let Some(lit_pos) = attr_str[pos..].find(lit_pattern) { + let start_pos = pos + lit_pos + lit_pattern.len(); + + // Find the closing quote of the literal + if let Some(quote_pos) = attr_str[start_pos..].find('\"') { + let world_name = &attr_str[start_pos..(start_pos + quote_pos)]; + println!("Extracted wit_world: {}", world_name); + return Ok(world_name.to_string()); + } + } + } + } + } + anyhow::bail!("wit_world not found in hyperprocess attribute") +} + +// Convert Rust type to WIT type, including downstream types +fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result { + match ty { + Type::Path(type_path) => { + if type_path.path.segments.is_empty() { + return Ok("unknown".to_string()); + } + + let ident = &type_path.path.segments.last().unwrap().ident; + let type_name = ident.to_string(); + + match type_name.as_str() { + "i32" => Ok("s32".to_string()), + "u32" => Ok("u32".to_string()), + "i64" => Ok("s64".to_string()), + "u64" => Ok("u64".to_string()), + "f32" => Ok("f32".to_string()), + "f64" => Ok("f64".to_string()), + "String" => Ok("string".to_string()), + "bool" => Ok("bool".to_string()), + "Vec" => { + if let syn::PathArguments::AngleBracketed(args) = + &type_path.path.segments.last().unwrap().arguments + { + if let Some(syn::GenericArgument::Type(inner_ty)) = args.args.first() { + let inner_type = rust_type_to_wit(inner_ty, used_types)?; + Ok(format!("list<{}>", inner_type)) + } else { + Ok("list".to_string()) + } + } else { + Ok("list".to_string()) + } + } + "Option" => { + if let syn::PathArguments::AngleBracketed(args) = + &type_path.path.segments.last().unwrap().arguments + { + if let Some(syn::GenericArgument::Type(inner_ty)) = args.args.first() { + let inner_type = rust_type_to_wit(inner_ty, used_types)?; + Ok(format!("option<{}>", inner_type)) + } else { + Ok("option".to_string()) + } + } else { + Ok("option".to_string()) + } + } + "HashMap" | "BTreeMap" => { + if let syn::PathArguments::AngleBracketed(args) = + &type_path.path.segments.last().unwrap().arguments + { + if args.args.len() >= 2 { + if let (Some(syn::GenericArgument::Type(key_ty)), Some(syn::GenericArgument::Type(val_ty))) = + (args.args.first(), args.args.get(1)) { + let key_type = rust_type_to_wit(key_ty, used_types)?; + let val_type = rust_type_to_wit(val_ty, used_types)?; + // For HashMaps, we'll generate a list of tuples where each tuple contains a key and value + Ok(format!("list>", key_type, val_type)) + } else { + Ok("list>".to_string()) + } + } else { + Ok("list>".to_string()) + } + } else { + Ok("list>".to_string()) + } + } + custom => { + // Validate custom type name + validate_name(custom, "Type")?; + + // Convert custom type to kebab-case and add to used types + let kebab_custom = to_kebab_case(custom); + used_types.insert(kebab_custom.clone()); + Ok(kebab_custom) + } + } + } + Type::Reference(type_ref) => { + // Handle references by using the underlying type + rust_type_to_wit(&type_ref.elem, used_types) + } + Type::Tuple(type_tuple) => { + if type_tuple.elems.is_empty() { + // Empty tuple is unit in WIT + Ok("unit".to_string()) + } else { + // Create a tuple representation in WIT + let mut elem_types = Vec::new(); + for elem in &type_tuple.elems { + elem_types.push(rust_type_to_wit(elem, used_types)?); + } + Ok(format!("tuple<{}>", elem_types.join(", "))) + } + } + _ => Ok("unknown".to_string()), + } +} + +// Find all Rust files in a crate directory +fn find_rust_files(crate_path: &Path) -> Vec { + let mut rust_files = Vec::new(); + let src_dir = crate_path.join("src"); + + println!("Finding Rust files in {}", src_dir.display()); + + if !src_dir.exists() || !src_dir.is_dir() { + println!("No src directory found at {}", src_dir.display()); + return rust_files; + } + + for entry in WalkDir::new(src_dir) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + if path.is_file() && path.extension().map_or(false, |ext| ext == "rs") { + println!("Found Rust file: {}", path.display()); + rust_files.push(path.to_path_buf()); + } + } + + println!("Found {} Rust files", rust_files.len()); + rust_files +} + +// Collect type definitions (structs and enums) from a file +fn collect_type_definitions_from_file(file_path: &Path) -> Result> { + println!("Collecting type definitions from file: {}", file_path.display()); + + let content = fs::read_to_string(file_path) + .with_context(|| format!("Failed to read file: {}", file_path.display()))?; + + let ast = syn::parse_file(&content) + .with_context(|| format!("Failed to parse file: {}", file_path.display()))?; + + let mut type_defs = HashMap::new(); + + for item in &ast.items { + match item { + Item::Struct(item_struct) => { + // Validate struct name doesn't contain numbers or "stream" + let orig_name = item_struct.ident.to_string(); + + // Skip trying to validate if name contains "__" as these are likely internal types + if orig_name.contains("__") { + println!(" Skipping likely internal struct: {}", orig_name); + continue; + } + + match validate_name(&orig_name, "Struct") { + Ok(_) => { + // Use kebab-case for struct name + let name = to_kebab_case(&orig_name); + println!(" Found struct: {} -> {}", orig_name, name); + + let fields: Vec = match &item_struct.fields { + syn::Fields::Named(fields) => { + let mut used_types = HashSet::new(); + let mut field_strings = Vec::new(); + + for f in &fields.named { + if let Some(field_ident) = &f.ident { + // Validate field name doesn't contain digits + let field_orig_name = field_ident.to_string(); + + match validate_name(&field_orig_name, "Field") { + Ok(_) => { + // Convert field names to kebab-case + let field_name = to_kebab_case(&field_orig_name); + + // Skip if field conversion failed + if field_name.is_empty() { + println!(" Skipping field with empty name conversion"); + continue; + } + + let field_type = match rust_type_to_wit(&f.ty, &mut used_types) { + Ok(ty) => ty, + Err(e) => { + println!(" Error converting field type: {}", e); + "unknown".to_string() + } + }; + + println!(" Field: {} -> {}", field_name, field_type); + field_strings.push(format!(" {}: {}", field_name, field_type)); + }, + Err(e) => { + println!(" Skipping field with invalid name: {}", e); + continue; + } + } + } + } + + field_strings + } + _ => Vec::new(), + }; + + if !fields.is_empty() { + type_defs.insert( + name.clone(), + format!(" record {} {{\n{}\n }}", name, fields.join(",\n")), + ); + } + }, + Err(e) => { + println!(" Skipping struct with invalid name: {}", e); + continue; + } + } + } + Item::Enum(item_enum) => { + // Validate enum name doesn't contain numbers or "stream" + let orig_name = item_enum.ident.to_string(); + + // Skip trying to validate if name contains "__" as these are likely internal types + if orig_name.contains("__") { + println!(" Skipping likely internal enum: {}", orig_name); + continue; + } + + match validate_name(&orig_name, "Enum") { + Ok(_) => { + // Use kebab-case for enum name + let name = to_kebab_case(&orig_name); + println!(" Found enum: {} -> {}", orig_name, name); + + let mut variants = Vec::new(); + let mut skip_enum = false; + + for v in &item_enum.variants { + let variant_orig_name = v.ident.to_string(); + + // Validate variant name + match validate_name(&variant_orig_name, "Enum variant") { + Ok(_) => { + match &v.fields { + syn::Fields::Unnamed(fields) if fields.unnamed.len() == 1 => { + let mut used_types = HashSet::new(); + + match rust_type_to_wit( + &fields.unnamed.first().unwrap().ty, + &mut used_types + ) { + Ok(ty) => { + // Use kebab-case for variant names and use parentheses for type + let variant_name = to_kebab_case(&variant_orig_name); + println!(" Variant: {} -> {}({})", variant_orig_name, variant_name, ty); + variants.push(format!(" {}({})", variant_name, ty)); + }, + Err(e) => { + println!(" Error converting variant type: {}", e); + skip_enum = true; + break; + } + } + } + syn::Fields::Unit => { + // Use kebab-case for variant names + let variant_name = to_kebab_case(&variant_orig_name); + println!(" Variant: {} -> {}", variant_orig_name, variant_name); + variants.push(format!(" {}", variant_name)); + }, + _ => { + println!(" Skipping complex variant: {}", variant_orig_name); + // Complex variants with multiple fields aren't directly supported in WIT + // For simplicity, we'll skip enums with complex variants + skip_enum = true; + break; + }, + } + }, + Err(e) => { + println!(" Skipping variant with invalid name: {}", e); + skip_enum = true; + break; + } + } + } + + if !skip_enum && !variants.is_empty() { + type_defs.insert( + name.clone(), + format!(" variant {} {{\n{}\n }}", name, variants.join(",\n")), + ); + } + }, + Err(e) => { + println!(" Skipping enum with invalid name: {}", e); + continue; + } + } + } + _ => {} + } + } + + println!("Collected {} type definitions from file", type_defs.len()); + Ok(type_defs) +} + +// Find all relevant Rust projects +fn find_rust_projects(base_dir: &Path) -> Vec { + let mut projects = Vec::new(); + println!("Scanning for Rust projects in {}", base_dir.display()); + + for entry in WalkDir::new(base_dir) + .max_depth(1) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + + if path.is_dir() && path != base_dir { + let cargo_toml = path.join("Cargo.toml"); + println!("Checking {}", cargo_toml.display()); + + if cargo_toml.exists() { + // Try to read and parse Cargo.toml + if let Ok(content) = fs::read_to_string(&cargo_toml) { + if let Ok(cargo_data) = content.parse::() { + // Check for the specific metadata + if let Some(metadata) = cargo_data + .get("package") + .and_then(|p| p.get("metadata")) + .and_then(|m| m.get("component")) + { + if let Some(package) = metadata.get("package") { + if let Some(package_str) = package.as_str() { + println!(" Found package.metadata.component.package = {:?}", package_str); + if package_str == "hyperware:process" { + println!(" Adding project: {}", path.display()); + projects.push(path.to_path_buf()); + } + } + } + } else { + println!(" No package.metadata.component metadata found"); + } + } + } + } + } + } + + println!("Found {} relevant Rust projects", projects.len()); + projects +} + +// Helper function to generate signature struct for specific attribute type +fn generate_signature_struct( + kebab_name: &str, + attr_type: &str, + method: &syn::ImplItemFn, + used_types: &mut HashSet, +) -> Result { + // Create signature struct name with attribute type + let signature_struct_name = format!("{}-signature-{}", kebab_name, attr_type); + + // Generate comment for this specific function + let comment = format!(" // Function signature for: {} ({})", kebab_name, attr_type); + + // Create struct fields that directly represent function parameters + let mut struct_fields = Vec::new(); + + // Add target parameter based on attribute type + if attr_type == "http" { + struct_fields.push(" target: string".to_string()); + } else { // remote or local + struct_fields.push(" target: address".to_string()); + } + + // Process function parameters (skip &self and &mut self) + for arg in &method.sig.inputs { + if let syn::FnArg::Typed(pat_type) = arg { + if let syn::Pat::Ident(pat_ident) = &*pat_type.pat { + // Skip &self and &mut self + if pat_ident.ident == "self" { + continue; + } + + // Get original param name and convert to kebab-case + let param_orig_name = pat_ident.ident.to_string(); + + // Validate parameter name + match validate_name(¶m_orig_name, "Parameter") { + Ok(_) => { + let param_name = to_kebab_case(¶m_orig_name); + + // Rust type to WIT type + match rust_type_to_wit(&pat_type.ty, used_types) { + Ok(param_type) => { + // Add field directly to the struct + struct_fields.push(format!(" {}: {}", param_name, param_type)); + }, + Err(e) => { + println!(" Error converting parameter type: {}", e); + // Use a placeholder type for this parameter + struct_fields.push(format!(" {}: unknown", param_name)); + } + } + }, + Err(e) => { + println!(" Skipping parameter with invalid name: {}", e); + // Use a placeholder for invalid parameter names + struct_fields.push(" invalid-param: unknown".to_string()); + } + } + } + } + } + + // Add return type field + match &method.sig.output { + syn::ReturnType::Type(_, ty) => { + match rust_type_to_wit(&*ty, used_types) { + Ok(return_type) => { + struct_fields.push(format!(" returning: {}", return_type)); + }, + Err(e) => { + println!(" Error converting return type: {}", e); + struct_fields.push(" returning: unknown".to_string()); + } + } + } + _ => { + // For unit return type + struct_fields.push(" returning: unit".to_string()); + } + } + + // Combine everything into a record definition + let record_def = format!( + "{}\n record {} {{\n{}\n }}", + comment, + signature_struct_name, + struct_fields.join(",\n") + ); + + Ok(record_def) +} + +// Helper trait to get TypePath from Type +trait AsTypePath { + fn as_type_path(&self) -> Option<&syn::TypePath>; +} + +impl AsTypePath for syn::Type { + fn as_type_path(&self) -> Option<&syn::TypePath> { + match self { + syn::Type::Path(tp) => Some(tp), + _ => None, + } + } +} + +// Process a single Rust project and generate WIT files +fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result> { + println!("\nProcessing project: {}", project_path.display()); + + // Find lib.rs for this project + let lib_rs = project_path.join("src").join("lib.rs"); + + if !lib_rs.exists() { + println!("No lib.rs found for project: {}", project_path.display()); + return Ok(None); + } + + // Find all Rust files in the project + let rust_files = find_rust_files(project_path); + + // Collect all type definitions from all Rust files + let mut all_type_defs = HashMap::new(); + for file_path in &rust_files { + match collect_type_definitions_from_file(file_path) { + Ok(file_type_defs) => { + for (name, def) in file_type_defs { + all_type_defs.insert(name, def); + } + }, + Err(e) => { + println!("Error collecting type definitions from {}: {}", file_path.display(), e); + // Continue with other files + } + } + } + + println!("Collected {} total type definitions", all_type_defs.len()); + + // Parse lib.rs to find the hyperprocess attribute and interface details + let lib_content = fs::read_to_string(&lib_rs) + .with_context(|| format!("Failed to read lib.rs for project: {}", project_path.display()))?; + + let ast = syn::parse_file(&lib_content) + .with_context(|| format!("Failed to parse lib.rs for project: {}", project_path.display()))?; + + let mut wit_world = None; + let mut interface_name = None; + let mut kebab_interface_name = None; + let mut impl_item_with_hyperprocess = None; + + println!("Scanning for impl blocks with hyperprocess attribute"); + for item in &ast.items { + if let Item::Impl(impl_item) = item { + // Check if this impl block has a #[hyperprocess] attribute + if let Some(attr) = impl_item.attrs.iter().find(|attr| attr.path().is_ident("hyperprocess")) { + println!("Found hyperprocess attribute"); + + // Extract the wit_world name + match extract_wit_world(&[attr.clone()]) { + Ok(world_name) => { + println!("Extracted wit_world: {}", world_name); + wit_world = Some(world_name); + + // Get the interface name from the impl type + interface_name = impl_item + .self_ty + .as_ref() + .as_type_path() + .map(|tp| { + if let Some(last_segment) = tp.path.segments.last() { + last_segment.ident.to_string() + } else { + "Unknown".to_string() + } + }); + + // Check for "State" suffix and remove it + if let Some(ref name) = interface_name { + // Validate the interface name + if let Err(e) = validate_name(name, "Interface") { + println!("Interface name validation failed: {}", e); + continue; + } + + // Remove State suffix if present + let base_name = remove_state_suffix(name); + + // Convert to kebab-case for file name and interface name + kebab_interface_name = Some(to_kebab_case(&base_name)); + + println!("Interface name: {:?}", interface_name); + println!("Base name: {}", base_name); + println!("Kebab interface name: {:?}", kebab_interface_name); + + // Save the impl item for later processing + impl_item_with_hyperprocess = Some(impl_item.clone()); + } + }, + Err(e) => println!("Failed to extract wit_world: {}", e), + } + } + } + } + + // Now generate the WIT content for the interface + if let (Some(ref iface_name), Some(ref kebab_name), Some(ref impl_item)) = + (&interface_name, &kebab_interface_name, &impl_item_with_hyperprocess) { + let mut signature_structs = Vec::new(); + let mut used_types = HashSet::new(); + + for item in &impl_item.items { + if let ImplItem::Fn(method) = item { + let method_name = method.sig.ident.to_string(); + println!(" Examining method: {}", method_name); + + // Check for attribute types + let has_remote = method.attrs.iter().any(|attr| attr.path().is_ident("remote")); + let has_local = method.attrs.iter().any(|attr| attr.path().is_ident("local")); + let has_http = method.attrs.iter().any(|attr| attr.path().is_ident("http")); + + if has_remote || has_local || has_http { + println!(" Has relevant attributes: remote={}, local={}, http={}", + has_remote, has_local, has_http); + + // Validate function name + match validate_name(&method_name, "Function") { + Ok(_) => { + // Convert function name to kebab-case + let kebab_name = to_kebab_case(&method_name); + println!(" Processing method: {} -> {}", method_name, kebab_name); + + // Generate a signature struct for each attribute type + if has_remote { + match generate_signature_struct(&kebab_name, "remote", method, &mut used_types) { + Ok(remote_struct) => signature_structs.push(remote_struct), + Err(e) => println!(" Error generating remote signature struct: {}", e), + } + } + + if has_local { + match generate_signature_struct(&kebab_name, "local", method, &mut used_types) { + Ok(local_struct) => signature_structs.push(local_struct), + Err(e) => println!(" Error generating local signature struct: {}", e), + } + } + + if has_http { + match generate_signature_struct(&kebab_name, "http", method, &mut used_types) { + Ok(http_struct) => signature_structs.push(http_struct), + Err(e) => println!(" Error generating HTTP signature struct: {}", e), + } + } + }, + Err(e) => { + println!(" Skipping method with invalid name: {}", e); + } + } + } else { + println!(" Skipping method without relevant attributes"); + } + } + } + + // Include all defined types, not just the ones used in interface functions + println!("Including all defined types ({})", all_type_defs.len()); + + // Convert all type definitions to a vector + let mut type_defs: Vec = all_type_defs.values().cloned().collect(); + + // Sort them for consistent output + type_defs.sort(); + + // Generate the final WIT content + if signature_structs.is_empty() { + println!("No functions found for interface {}", iface_name); + } else { + // Start with the interface comment + let mut content = " // This interface contains function signature definitions that will be used\n // by the hyper-bindgen macro to generate async function bindings.\n //\n // NOTE: This is currently a hacky workaround since WIT async functions are not\n // available until WASI Preview 3. Once Preview 3 is integrated into Hyperware,\n // we should switch to using proper async WIT function signatures instead of\n // this struct-based approach with hyper-bindgen generating the async stubs.\n".to_string(); + + // Add standard imports + content.push_str("\n use standard.{address};\n\n"); + + // Add type definitions if any + if !type_defs.is_empty() { + content.push_str(&type_defs.join("\n\n")); + content.push_str("\n\n"); + } + + // Add signature structs + content.push_str(&signature_structs.join("\n\n")); + + // Wrap in interface block + let final_content = format!("interface {} {{\n{}\n}}\n", kebab_name, content); + println!("Generated interface content for {} with {} signature structs", iface_name, signature_structs.len()); + + // Write the interface file with kebab-case name + let interface_file = api_dir.join(format!("{}.wit", kebab_name)); + println!("Writing WIT file to {}", interface_file.display()); + + fs::write(&interface_file, &final_content) + .with_context(|| format!("Failed to write {}", interface_file.display()))?; + + println!("Successfully wrote WIT file"); + } + } + + if let (Some(_), Some(_), Some(kebab_iface)) = (wit_world, interface_name, kebab_interface_name) { + println!("Returning import statement for interface {}", kebab_iface); + // Use kebab-case interface name for import + Ok(Some(format!(" import {};", kebab_iface))) + } else { + println!("No valid interface found"); + Ok(None) + } +} + +// Generate WIT files from Rust code +pub fn generate_wit_files(base_dir: &Path, api_dir: &Path) -> Result<(Vec, Vec)> { + // Find all relevant Rust projects + let projects = find_rust_projects(base_dir); + let mut processed_projects = Vec::new(); + + if projects.is_empty() { + println!("No relevant Rust projects found."); + return Ok((Vec::new(), Vec::new())); + } + + // Process each project and collect world imports + let mut new_imports = Vec::new(); + let mut interfaces = Vec::new(); + + for project_path in &projects { + println!("Processing project: {}", project_path.display()); + + match process_rust_project(project_path, api_dir) { + Ok(Some(import)) => { + println!("Got import statement: {}", import); + new_imports.push(import.clone()); + + // Extract interface name from import statement + let interface_name = import + .trim_start_matches(" import ") + .trim_end_matches(";") + .to_string(); + + interfaces.push(interface_name); + processed_projects.push(project_path.clone()); + }, + Ok(None) => println!("No import statement generated"), + Err(e) => println!("Error processing project: {}", e), + } + } + + println!("Collected {} new imports", new_imports.len()); + + // Check for existing world definition files and update them + println!("Looking for existing world definition files"); + let mut updated_world = false; + + for entry in WalkDir::new(api_dir) + .max_depth(1) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { + println!("Checking WIT file: {}", path.display()); + + if let Ok(content) = fs::read_to_string(path) { + if content.contains("world ") { + println!("Found world definition file"); + + // Extract the world name and existing imports + let lines: Vec<&str> = content.lines().collect(); + let mut world_name = None; + let mut existing_imports = Vec::new(); + let mut include_line = " include process-v1;".to_string(); + + for line in &lines { + let trimmed = line.trim(); + + if trimmed.starts_with("world ") { + if let Some(name) = trimmed.split_whitespace().nth(1) { + world_name = Some(name.trim_end_matches(" {").to_string()); + } + } else if trimmed.starts_with("import ") { + existing_imports.push(trimmed.to_string()); + } else if trimmed.starts_with("include ") { + include_line = trimmed.to_string(); + } + } + + if let Some(world_name) = world_name { + println!("Extracted world name: {}", world_name); + + // Determine the include line based on world name + // If world name starts with "types-", use "include lib;" instead + if world_name.starts_with("types-") { + include_line = " include lib;".to_string(); + } else { + // Keep existing include or default to process-v1 + if !include_line.contains("include ") { + include_line = " include process-v1;".to_string(); + } + } + + // Combine existing imports with new imports + let mut all_imports = existing_imports.clone(); + + for import in &new_imports { + let import_stmt = import.trim(); + if !all_imports.iter().any(|i| i.trim() == import_stmt) { + all_imports.push(import_stmt.to_string()); + } + } + + // Make sure all imports have proper indentation + let all_imports_with_indent: Vec = all_imports + .iter() + .map(|import| { + if import.starts_with(" ") { + import.clone() + } else { + format!(" {}", import.trim()) + } + }) + .collect(); + + let imports_section = all_imports_with_indent.join("\n"); + + // Create updated world content with proper indentation + let world_content = format!( + "world {} {{\n{}\n {}\n}}", + world_name, + imports_section, + include_line.trim() + ); + + println!("Writing updated world definition to {}", path.display()); + // Write the updated world file + fs::write(path, world_content) + .with_context(|| format!("Failed to write updated world file: {}", path.display()))?; + + println!("Successfully updated world definition"); + updated_world = true; + } + } + } + } + } + + // If no world definitions were found, create a default one + if !updated_world && !new_imports.is_empty() { + // Define default world name + let default_world = "async-app-template-dot-os-v0"; + println!("No existing world definitions found, creating default with name: {}", default_world); + + // Create world content with process-v1 include and proper indentation for imports + let imports_with_indent: Vec = new_imports + .iter() + .map(|import| { + if import.starts_with(" ") { + import.clone() + } else { + format!(" {}", import.trim()) + } + }) + .collect(); + + // Determine include based on world name + let include_line = if default_world.starts_with("types-") { + "include lib;" + } else { + "include process-v1;" + }; + + let world_content = format!( + "world {} {{\n{}\n {}\n}}", + default_world, + imports_with_indent.join("\n"), + include_line + ); + + let world_file = api_dir.join(format!("{}.wit", default_world)); + println!("Writing default world definition to {}", world_file.display()); + + fs::write(&world_file, world_content) + .with_context(|| format!("Failed to write default world file: {}", world_file.display()))?; + + println!("Successfully created default world definition"); + } + + println!("WIT files generated successfully in the 'api' directory."); + Ok((processed_projects, interfaces)) +} \ No newline at end of file From 302aef0e0b8e2c7c9aca897e9b6baabc93228e98 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 3 Apr 2025 17:21:16 -0700 Subject: [PATCH 2/8] build: add minimal use of wit_generator & get it compiling --- Cargo.toml | 3 +- src/build/mod.rs | 14 + src/build/wit_generator.rs | 550 ++++++++++++++++++++------------- src/build_start_package/mod.rs | 2 + src/main.rs | 18 ++ src/run_tests/mod.rs | 4 + 6 files changed, 375 insertions(+), 216 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 88c31954..7c6f1998 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,8 +52,7 @@ semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10.8" -syn = { version = "2.0", features = ["full", "visit", "extra-traits"] } -#syn = { version = "2.0", features = ["full", "visit"] } +syn = { version = "2.0", features = ["full", "visit", "parsing", "extra-traits"] } thiserror = "1.0" tokio = { version = "1.28", features = [ "macros", diff --git a/src/build/mod.rs b/src/build/mod.rs index 9e2382e3..b8059fc7 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -32,6 +32,8 @@ use crate::KIT_CACHE; mod rewrite; use rewrite::copy_and_rewrite_package; +mod wit_generator; + const PY_VENV_NAME: &str = "process_env"; const JAVASCRIPT_SRC_PATH: &str = "src/lib.js"; const PYTHON_SRC_PATH: &str = "src/lib.py"; @@ -1161,6 +1163,7 @@ async fn fetch_dependencies( include: &HashSet, exclude: &HashSet, rewrite: bool, + hyperapp: bool, force: bool, verbose: bool, ) -> Result<()> { @@ -1178,6 +1181,7 @@ async fn fetch_dependencies( vec![], // TODO: what about deps-of-deps? vec![], rewrite, + hyperapp, false, force, verbose, @@ -1215,6 +1219,7 @@ async fn fetch_dependencies( local_dep_deps, vec![], rewrite, + hyperapp, false, force, verbose, @@ -1531,6 +1536,7 @@ async fn compile_package( include: &HashSet, exclude: &HashSet, rewrite: bool, + hyperapp: bool, force: bool, verbose: bool, ignore_deps: bool, // for internal use; may cause problems when adding recursive deps @@ -1554,6 +1560,7 @@ async fn compile_package( include, exclude, rewrite, + hyperapp, force, verbose, ) @@ -1661,6 +1668,7 @@ pub async fn execute( local_dependencies: Vec, add_paths_to_api: Vec, rewrite: bool, + hyperapp: bool, reproducible: bool, force: bool, verbose: bool, @@ -1753,6 +1761,11 @@ pub async fn execute( copy_and_rewrite_package(package_dir)? }; + if hyperapp { + let api_dir = live_dir.join("api"); + let (_processed_projects, _interfaces) = wit_generator::generate_wit_files(&live_dir, &api_dir)?; + } + let ui_dirs = get_ui_dirs(&live_dir, &include, &exclude)?; if !no_ui && !ui_dirs.is_empty() { if !skip_deps_check { @@ -1779,6 +1792,7 @@ pub async fn execute( &include, &exclude, rewrite, + hyperapp, force, verbose, ignore_deps, diff --git a/src/build/wit_generator.rs b/src/build/wit_generator.rs index e5ad6949..43fffb6e 100644 --- a/src/build/wit_generator.rs +++ b/src/build/wit_generator.rs @@ -1,10 +1,15 @@ -use anyhow::{Context, Result}; +//use anyhow::{Context}; use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; + +use color_eyre::{ + eyre::{bail, WrapErr}, + Result, +}; use syn::{self, Attribute, ImplItem, Item, Type}; -use walkdir::WalkDir; use toml::Value; +use walkdir::WalkDir; // Helper functions for naming conventions fn to_kebab_case(s: &str) -> String { @@ -12,19 +17,19 @@ fn to_kebab_case(s: &str) -> String { if s.contains('_') { return s.replace('_', "-"); } - + let mut result = String::with_capacity(s.len() + 5); // Extra capacity for hyphens let chars: Vec = s.chars().collect(); - + for (i, &c) in chars.iter().enumerate() { if c.is_uppercase() { // Add hyphen if: // 1. Not the first character // 2. Previous character is lowercase // 3. Or next character is lowercase (to handle acronyms like HTML) - if i > 0 && - (chars[i-1].is_lowercase() || - (i < chars.len() - 1 && chars[i+1].is_lowercase())) + if i > 0 + && (chars[i - 1].is_lowercase() + || (i < chars.len() - 1 && chars[i + 1].is_lowercase())) { result.push('-'); } @@ -33,7 +38,7 @@ fn to_kebab_case(s: &str) -> String { result.push(c); } } - + result } @@ -41,14 +46,22 @@ fn to_kebab_case(s: &str) -> String { fn validate_name(name: &str, kind: &str) -> Result<()> { // Check for numbers if name.chars().any(|c| c.is_digit(10)) { - anyhow::bail!("Error: {} name '{}' contains numbers, which is not allowed", kind, name); + bail!( + "Error: {} name '{}' contains numbers, which is not allowed", + kind, + name + ); } - + // Check for "stream" if name.to_lowercase().contains("stream") { - anyhow::bail!("Error: {} name '{}' contains 'stream', which is not allowed", kind, name); + bail!( + "Error: {} name '{}' contains 'stream', which is not allowed", + kind, + name + ); } - + Ok(()) } @@ -56,7 +69,7 @@ fn validate_name(name: &str, kind: &str) -> Result<()> { fn remove_state_suffix(name: &str) -> String { if name.ends_with("State") { let len = name.len(); - return name[0..len-5].to_string(); + return name[0..len - 5].to_string(); } name.to_string() } @@ -68,16 +81,16 @@ fn extract_wit_world(attrs: &[Attribute]) -> Result { // Convert attribute to string representation let attr_str = format!("{:?}", attr); println!("Attribute string: {}", attr_str); - + // Look for wit_world in the attribute string if let Some(pos) = attr_str.find("wit_world") { println!("Found wit_world at position {}", pos); - + // Find the literal value after wit_world by looking for lit: "value" let lit_pattern = "lit: \""; if let Some(lit_pos) = attr_str[pos..].find(lit_pattern) { let start_pos = pos + lit_pos + lit_pattern.len(); - + // Find the closing quote of the literal if let Some(quote_pos) = attr_str[start_pos..].find('\"') { let world_name = &attr_str[start_pos..(start_pos + quote_pos)]; @@ -88,7 +101,7 @@ fn extract_wit_world(attrs: &[Attribute]) -> Result { } } } - anyhow::bail!("wit_world not found in hyperprocess attribute") + bail!("wit_world not found in hyperprocess attribute") } // Convert Rust type to WIT type, including downstream types @@ -98,10 +111,10 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result Ok("s32".to_string()), "u32" => Ok("u32".to_string()), @@ -112,7 +125,7 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result Ok("string".to_string()), "bool" => Ok("bool".to_string()), "Vec" => { - if let syn::PathArguments::AngleBracketed(args) = + if let syn::PathArguments::AngleBracketed(args) = &type_path.path.segments.last().unwrap().arguments { if let Some(syn::GenericArgument::Type(inner_ty)) = args.args.first() { @@ -144,8 +157,11 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result= 2 { - if let (Some(syn::GenericArgument::Type(key_ty)), Some(syn::GenericArgument::Type(val_ty))) = - (args.args.first(), args.args.get(1)) { + if let ( + Some(syn::GenericArgument::Type(key_ty)), + Some(syn::GenericArgument::Type(val_ty)), + ) = (args.args.first(), args.args.get(1)) + { let key_type = rust_type_to_wit(key_ty, used_types)?; let val_type = rust_type_to_wit(val_ty, used_types)?; // For HashMaps, we'll generate a list of tuples where each tuple contains a key and value @@ -163,7 +179,7 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result { // Validate custom type name validate_name(custom, "Type")?; - + // Convert custom type to kebab-case and add to used types let kebab_custom = to_kebab_case(custom); used_types.insert(kebab_custom.clone()); @@ -196,111 +212,126 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result Vec { let mut rust_files = Vec::new(); let src_dir = crate_path.join("src"); - + println!("Finding Rust files in {}", src_dir.display()); - + if !src_dir.exists() || !src_dir.is_dir() { println!("No src directory found at {}", src_dir.display()); return rust_files; } - - for entry in WalkDir::new(src_dir) - .into_iter() - .filter_map(Result::ok) - { + + for entry in WalkDir::new(src_dir).into_iter().filter_map(Result::ok) { let path = entry.path(); if path.is_file() && path.extension().map_or(false, |ext| ext == "rs") { println!("Found Rust file: {}", path.display()); rust_files.push(path.to_path_buf()); } } - + println!("Found {} Rust files", rust_files.len()); rust_files } // Collect type definitions (structs and enums) from a file fn collect_type_definitions_from_file(file_path: &Path) -> Result> { - println!("Collecting type definitions from file: {}", file_path.display()); - + println!( + "Collecting type definitions from file: {}", + file_path.display() + ); + let content = fs::read_to_string(file_path) .with_context(|| format!("Failed to read file: {}", file_path.display()))?; - + let ast = syn::parse_file(&content) .with_context(|| format!("Failed to parse file: {}", file_path.display()))?; - + let mut type_defs = HashMap::new(); - + for item in &ast.items { match item { Item::Struct(item_struct) => { // Validate struct name doesn't contain numbers or "stream" let orig_name = item_struct.ident.to_string(); - + // Skip trying to validate if name contains "__" as these are likely internal types if orig_name.contains("__") { println!(" Skipping likely internal struct: {}", orig_name); continue; } - + match validate_name(&orig_name, "Struct") { Ok(_) => { // Use kebab-case for struct name let name = to_kebab_case(&orig_name); println!(" Found struct: {} -> {}", orig_name, name); - + let fields: Vec = match &item_struct.fields { syn::Fields::Named(fields) => { let mut used_types = HashSet::new(); let mut field_strings = Vec::new(); - + for f in &fields.named { if let Some(field_ident) = &f.ident { // Validate field name doesn't contain digits let field_orig_name = field_ident.to_string(); - + match validate_name(&field_orig_name, "Field") { Ok(_) => { // Convert field names to kebab-case let field_name = to_kebab_case(&field_orig_name); - + // Skip if field conversion failed if field_name.is_empty() { println!(" Skipping field with empty name conversion"); continue; } - - let field_type = match rust_type_to_wit(&f.ty, &mut used_types) { + + let field_type = match rust_type_to_wit( + &f.ty, + &mut used_types, + ) { Ok(ty) => ty, Err(e) => { - println!(" Error converting field type: {}", e); + println!( + " Error converting field type: {}", + e + ); "unknown".to_string() } }; - - println!(" Field: {} -> {}", field_name, field_type); - field_strings.push(format!(" {}: {}", field_name, field_type)); - }, + + println!( + " Field: {} -> {}", + field_name, field_type + ); + field_strings.push(format!( + " {}: {}", + field_name, field_type + )); + } Err(e) => { - println!(" Skipping field with invalid name: {}", e); + println!( + " Skipping field with invalid name: {}", + e + ); continue; } } } } - + field_strings } _ => Vec::new(), }; - + if !fields.is_empty() { type_defs.insert( name.clone(), format!(" record {} {{\n{}\n }}", name, fields.join(",\n")), ); } - }, + } Err(e) => { println!(" Skipping struct with invalid name: {}", e); continue; @@ -310,44 +341,56 @@ fn collect_type_definitions_from_file(file_path: &Path) -> Result { // Validate enum name doesn't contain numbers or "stream" let orig_name = item_enum.ident.to_string(); - + // Skip trying to validate if name contains "__" as these are likely internal types if orig_name.contains("__") { println!(" Skipping likely internal enum: {}", orig_name); continue; } - + match validate_name(&orig_name, "Enum") { Ok(_) => { // Use kebab-case for enum name let name = to_kebab_case(&orig_name); println!(" Found enum: {} -> {}", orig_name, name); - + let mut variants = Vec::new(); let mut skip_enum = false; - + for v in &item_enum.variants { let variant_orig_name = v.ident.to_string(); - + // Validate variant name match validate_name(&variant_orig_name, "Enum variant") { Ok(_) => { match &v.fields { - syn::Fields::Unnamed(fields) if fields.unnamed.len() == 1 => { + syn::Fields::Unnamed(fields) + if fields.unnamed.len() == 1 => + { let mut used_types = HashSet::new(); - + match rust_type_to_wit( &fields.unnamed.first().unwrap().ty, - &mut used_types + &mut used_types, ) { Ok(ty) => { // Use kebab-case for variant names and use parentheses for type - let variant_name = to_kebab_case(&variant_orig_name); - println!(" Variant: {} -> {}({})", variant_orig_name, variant_name, ty); - variants.push(format!(" {}({})", variant_name, ty)); - }, + let variant_name = + to_kebab_case(&variant_orig_name); + println!( + " Variant: {} -> {}({})", + variant_orig_name, variant_name, ty + ); + variants.push(format!( + " {}({})", + variant_name, ty + )); + } Err(e) => { - println!(" Error converting variant type: {}", e); + println!( + " Error converting variant type: {}", + e + ); skip_enum = true; break; } @@ -356,18 +399,24 @@ fn collect_type_definitions_from_file(file_path: &Path) -> Result { // Use kebab-case for variant names let variant_name = to_kebab_case(&variant_orig_name); - println!(" Variant: {} -> {}", variant_orig_name, variant_name); + println!( + " Variant: {} -> {}", + variant_orig_name, variant_name + ); variants.push(format!(" {}", variant_name)); - }, + } _ => { - println!(" Skipping complex variant: {}", variant_orig_name); + println!( + " Skipping complex variant: {}", + variant_orig_name + ); // Complex variants with multiple fields aren't directly supported in WIT // For simplicity, we'll skip enums with complex variants skip_enum = true; break; - }, + } } - }, + } Err(e) => { println!(" Skipping variant with invalid name: {}", e); skip_enum = true; @@ -375,14 +424,18 @@ fn collect_type_definitions_from_file(file_path: &Path) -> Result { println!(" Skipping enum with invalid name: {}", e); continue; @@ -392,7 +445,7 @@ fn collect_type_definitions_from_file(file_path: &Path) -> Result {} } } - + println!("Collected {} type definitions from file", type_defs.len()); Ok(type_defs) } @@ -401,18 +454,18 @@ fn collect_type_definitions_from_file(file_path: &Path) -> Result Vec { let mut projects = Vec::new(); println!("Scanning for Rust projects in {}", base_dir.display()); - + for entry in WalkDir::new(base_dir) .max_depth(1) .into_iter() .filter_map(Result::ok) { let path = entry.path(); - + if path.is_dir() && path != base_dir { let cargo_toml = path.join("Cargo.toml"); println!("Checking {}", cargo_toml.display()); - + if cargo_toml.exists() { // Try to read and parse Cargo.toml if let Ok(content) = fs::read_to_string(&cargo_toml) { @@ -425,7 +478,10 @@ fn find_rust_projects(base_dir: &Path) -> Vec { { if let Some(package) = metadata.get("package") { if let Some(package_str) = package.as_str() { - println!(" Found package.metadata.component.package = {:?}", package_str); + println!( + " Found package.metadata.component.package = {:?}", + package_str + ); if package_str == "hyperware:process" { println!(" Adding project: {}", path.display()); projects.push(path.to_path_buf()); @@ -440,7 +496,7 @@ fn find_rust_projects(base_dir: &Path) -> Vec { } } } - + println!("Found {} relevant Rust projects", projects.len()); projects } @@ -454,20 +510,24 @@ fn generate_signature_struct( ) -> Result { // Create signature struct name with attribute type let signature_struct_name = format!("{}-signature-{}", kebab_name, attr_type); - + // Generate comment for this specific function - let comment = format!(" // Function signature for: {} ({})", kebab_name, attr_type); - + let comment = format!( + " // Function signature for: {} ({})", + kebab_name, attr_type + ); + // Create struct fields that directly represent function parameters let mut struct_fields = Vec::new(); - + // Add target parameter based on attribute type if attr_type == "http" { struct_fields.push(" target: string".to_string()); - } else { // remote or local + } else { + // remote or local struct_fields.push(" target: address".to_string()); } - + // Process function parameters (skip &self and &mut self) for arg in &method.sig.inputs { if let syn::FnArg::Typed(pat_type) = arg { @@ -476,28 +536,29 @@ fn generate_signature_struct( if pat_ident.ident == "self" { continue; } - + // Get original param name and convert to kebab-case let param_orig_name = pat_ident.ident.to_string(); - + // Validate parameter name match validate_name(¶m_orig_name, "Parameter") { Ok(_) => { let param_name = to_kebab_case(¶m_orig_name); - + // Rust type to WIT type match rust_type_to_wit(&pat_type.ty, used_types) { Ok(param_type) => { // Add field directly to the struct - struct_fields.push(format!(" {}: {}", param_name, param_type)); - }, + struct_fields + .push(format!(" {}: {}", param_name, param_type)); + } Err(e) => { println!(" Error converting parameter type: {}", e); // Use a placeholder type for this parameter struct_fields.push(format!(" {}: unknown", param_name)); } } - }, + } Err(e) => { println!(" Skipping parameter with invalid name: {}", e); // Use a placeholder for invalid parameter names @@ -507,26 +568,24 @@ fn generate_signature_struct( } } } - + // Add return type field match &method.sig.output { - syn::ReturnType::Type(_, ty) => { - match rust_type_to_wit(&*ty, used_types) { - Ok(return_type) => { - struct_fields.push(format!(" returning: {}", return_type)); - }, - Err(e) => { - println!(" Error converting return type: {}", e); - struct_fields.push(" returning: unknown".to_string()); - } + syn::ReturnType::Type(_, ty) => match rust_type_to_wit(&*ty, used_types) { + Ok(return_type) => { + struct_fields.push(format!(" returning: {}", return_type)); } - } + Err(e) => { + println!(" Error converting return type: {}", e); + struct_fields.push(" returning: unknown".to_string()); + } + }, _ => { // For unit return type struct_fields.push(" returning: unit".to_string()); } } - + // Combine everything into a record definition let record_def = format!( "{}\n record {} {{\n{}\n }}", @@ -534,7 +593,7 @@ fn generate_signature_struct( signature_struct_name, struct_fields.join(",\n") ); - + Ok(record_def) } @@ -555,18 +614,18 @@ impl AsTypePath for syn::Type { // Process a single Rust project and generate WIT files fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result> { println!("\nProcessing project: {}", project_path.display()); - + // Find lib.rs for this project let lib_rs = project_path.join("src").join("lib.rs"); - + if !lib_rs.exists() { println!("No lib.rs found for project: {}", project_path.display()); return Ok(None); } - + // Find all Rust files in the project let rust_files = find_rust_files(project_path); - + // Collect all type definitions from all Rust files let mut all_type_defs = HashMap::new(); for file_path in &rust_files { @@ -575,54 +634,66 @@ fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result { - println!("Error collecting type definitions from {}: {}", file_path.display(), e); + println!( + "Error collecting type definitions from {}: {}", + file_path.display(), + e + ); // Continue with other files } } } - + println!("Collected {} total type definitions", all_type_defs.len()); - + // Parse lib.rs to find the hyperprocess attribute and interface details - let lib_content = fs::read_to_string(&lib_rs) - .with_context(|| format!("Failed to read lib.rs for project: {}", project_path.display()))?; - - let ast = syn::parse_file(&lib_content) - .with_context(|| format!("Failed to parse lib.rs for project: {}", project_path.display()))?; - + let lib_content = fs::read_to_string(&lib_rs).with_context(|| { + format!( + "Failed to read lib.rs for project: {}", + project_path.display() + ) + })?; + + let ast = syn::parse_file(&lib_content).with_context(|| { + format!( + "Failed to parse lib.rs for project: {}", + project_path.display() + ) + })?; + let mut wit_world = None; let mut interface_name = None; let mut kebab_interface_name = None; let mut impl_item_with_hyperprocess = None; - + println!("Scanning for impl blocks with hyperprocess attribute"); for item in &ast.items { if let Item::Impl(impl_item) = item { // Check if this impl block has a #[hyperprocess] attribute - if let Some(attr) = impl_item.attrs.iter().find(|attr| attr.path().is_ident("hyperprocess")) { + if let Some(attr) = impl_item + .attrs + .iter() + .find(|attr| attr.path().is_ident("hyperprocess")) + { println!("Found hyperprocess attribute"); - + // Extract the wit_world name match extract_wit_world(&[attr.clone()]) { Ok(world_name) => { println!("Extracted wit_world: {}", world_name); wit_world = Some(world_name); - + // Get the interface name from the impl type - interface_name = impl_item - .self_ty - .as_ref() - .as_type_path() - .map(|tp| { - if let Some(last_segment) = tp.path.segments.last() { - last_segment.ident.to_string() - } else { - "Unknown".to_string() - } - }); - + interface_name = impl_item.self_ty.as_ref().as_type_path().map(|tp| { + if let Some(last_segment) = tp.path.segments.last() { + last_segment.ident.to_string() + } else { + "Unknown".to_string() + } + }); + // Check for "State" suffix and remove it if let Some(ref name) = interface_name { // Validate the interface name @@ -630,76 +701,111 @@ fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result println!("Failed to extract wit_world: {}", e), } } } } - + // Now generate the WIT content for the interface - if let (Some(ref iface_name), Some(ref kebab_name), Some(ref impl_item)) = - (&interface_name, &kebab_interface_name, &impl_item_with_hyperprocess) { + if let (Some(ref iface_name), Some(ref kebab_name), Some(ref impl_item)) = ( + &interface_name, + &kebab_interface_name, + &impl_item_with_hyperprocess, + ) { let mut signature_structs = Vec::new(); let mut used_types = HashSet::new(); - + for item in &impl_item.items { if let ImplItem::Fn(method) = item { let method_name = method.sig.ident.to_string(); println!(" Examining method: {}", method_name); - + // Check for attribute types - let has_remote = method.attrs.iter().any(|attr| attr.path().is_ident("remote")); - let has_local = method.attrs.iter().any(|attr| attr.path().is_ident("local")); + let has_remote = method + .attrs + .iter() + .any(|attr| attr.path().is_ident("remote")); + let has_local = method + .attrs + .iter() + .any(|attr| attr.path().is_ident("local")); let has_http = method.attrs.iter().any(|attr| attr.path().is_ident("http")); - + if has_remote || has_local || has_http { - println!(" Has relevant attributes: remote={}, local={}, http={}", - has_remote, has_local, has_http); - + println!( + " Has relevant attributes: remote={}, local={}, http={}", + has_remote, has_local, has_http + ); + // Validate function name match validate_name(&method_name, "Function") { Ok(_) => { // Convert function name to kebab-case let kebab_name = to_kebab_case(&method_name); println!(" Processing method: {} -> {}", method_name, kebab_name); - + // Generate a signature struct for each attribute type if has_remote { - match generate_signature_struct(&kebab_name, "remote", method, &mut used_types) { + match generate_signature_struct( + &kebab_name, + "remote", + method, + &mut used_types, + ) { Ok(remote_struct) => signature_structs.push(remote_struct), - Err(e) => println!(" Error generating remote signature struct: {}", e), + Err(e) => println!( + " Error generating remote signature struct: {}", + e + ), } } - + if has_local { - match generate_signature_struct(&kebab_name, "local", method, &mut used_types) { + match generate_signature_struct( + &kebab_name, + "local", + method, + &mut used_types, + ) { Ok(local_struct) => signature_structs.push(local_struct), - Err(e) => println!(" Error generating local signature struct: {}", e), + Err(e) => println!( + " Error generating local signature struct: {}", + e + ), } } - + if has_http { - match generate_signature_struct(&kebab_name, "http", method, &mut used_types) { + match generate_signature_struct( + &kebab_name, + "http", + method, + &mut used_types, + ) { Ok(http_struct) => signature_structs.push(http_struct), - Err(e) => println!(" Error generating HTTP signature struct: {}", e), + Err(e) => println!( + " Error generating HTTP signature struct: {}", + e + ), } } - }, + } Err(e) => { println!(" Skipping method with invalid name: {}", e); } @@ -709,51 +815,56 @@ fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result = all_type_defs.values().cloned().collect(); - + // Sort them for consistent output type_defs.sort(); - + // Generate the final WIT content if signature_structs.is_empty() { println!("No functions found for interface {}", iface_name); } else { // Start with the interface comment let mut content = " // This interface contains function signature definitions that will be used\n // by the hyper-bindgen macro to generate async function bindings.\n //\n // NOTE: This is currently a hacky workaround since WIT async functions are not\n // available until WASI Preview 3. Once Preview 3 is integrated into Hyperware,\n // we should switch to using proper async WIT function signatures instead of\n // this struct-based approach with hyper-bindgen generating the async stubs.\n".to_string(); - + // Add standard imports content.push_str("\n use standard.{address};\n\n"); - + // Add type definitions if any if !type_defs.is_empty() { content.push_str(&type_defs.join("\n\n")); content.push_str("\n\n"); } - + // Add signature structs content.push_str(&signature_structs.join("\n\n")); - + // Wrap in interface block let final_content = format!("interface {} {{\n{}\n}}\n", kebab_name, content); - println!("Generated interface content for {} with {} signature structs", iface_name, signature_structs.len()); - + println!( + "Generated interface content for {} with {} signature structs", + iface_name, + signature_structs.len() + ); + // Write the interface file with kebab-case name let interface_file = api_dir.join(format!("{}.wit", kebab_name)); println!("Writing WIT file to {}", interface_file.display()); - + fs::write(&interface_file, &final_content) .with_context(|| format!("Failed to write {}", interface_file.display()))?; - + println!("Successfully wrote WIT file"); } } - - if let (Some(_), Some(_), Some(kebab_iface)) = (wit_world, interface_name, kebab_interface_name) { + + if let (Some(_), Some(_), Some(kebab_iface)) = (wit_world, interface_name, kebab_interface_name) + { println!("Returning import statement for interface {}", kebab_iface); // Use kebab-case interface name for import Ok(Some(format!(" import {};", kebab_iface))) @@ -768,67 +879,67 @@ pub fn generate_wit_files(base_dir: &Path, api_dir: &Path) -> Result<(Vec { println!("Got import statement: {}", import); new_imports.push(import.clone()); - + // Extract interface name from import statement let interface_name = import .trim_start_matches(" import ") .trim_end_matches(";") .to_string(); - + interfaces.push(interface_name); processed_projects.push(project_path.clone()); - }, + } Ok(None) => println!("No import statement generated"), Err(e) => println!("Error processing project: {}", e), } } - + println!("Collected {} new imports", new_imports.len()); - + // Check for existing world definition files and update them println!("Looking for existing world definition files"); let mut updated_world = false; - + for entry in WalkDir::new(api_dir) .max_depth(1) .into_iter() .filter_map(Result::ok) { let path = entry.path(); - + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { println!("Checking WIT file: {}", path.display()); - + if let Ok(content) = fs::read_to_string(path) { if content.contains("world ") { println!("Found world definition file"); - + // Extract the world name and existing imports let lines: Vec<&str> = content.lines().collect(); let mut world_name = None; let mut existing_imports = Vec::new(); let mut include_line = " include process-v1;".to_string(); - + for line in &lines { let trimmed = line.trim(); - + if trimmed.starts_with("world ") { if let Some(name) = trimmed.split_whitespace().nth(1) { world_name = Some(name.trim_end_matches(" {").to_string()); @@ -839,10 +950,10 @@ pub fn generate_wit_files(base_dir: &Path, api_dir: &Path) -> Result<(Vec Result<(Vec = all_imports .iter() @@ -875,9 +986,9 @@ pub fn generate_wit_files(base_dir: &Path, api_dir: &Path) -> Result<(Vec Result<(Vec Result<(Vec = new_imports .iter() @@ -916,30 +1031,37 @@ pub fn generate_wit_files(base_dir: &Path, api_dir: &Path) -> Result<(Vec, add_paths_to_api: Vec, rewrite: bool, + hyperapp: bool, reproducible: bool, force: bool, verbose: bool, @@ -40,6 +41,7 @@ pub async fn execute( local_dependencies, add_paths_to_api, rewrite, + hyperapp, reproducible, force, verbose, diff --git a/src/main.rs b/src/main.rs index 87574074..758cde2f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -235,6 +235,7 @@ async fn execute( .map(|s| PathBuf::from(s)) .collect(); let rewrite = matches.get_one::("REWRITE").unwrap(); + let hyperapp = matches.get_one::("HYPERAPP").unwrap(); let reproducible = matches.get_one::("REPRODUCIBLE").unwrap(); let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); @@ -253,6 +254,7 @@ async fn execute( local_dependencies, add_paths_to_api, *rewrite, + *hyperapp, *reproducible, *force, *verbose, @@ -298,6 +300,7 @@ async fn execute( .map(|s| PathBuf::from(s)) .collect(); let rewrite = matches.get_one::("REWRITE").unwrap(); + let hyperapp = matches.get_one::("HYPERAPP").unwrap(); let reproducible = matches.get_one::("REPRODUCIBLE").unwrap(); let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); @@ -316,6 +319,7 @@ async fn execute( local_dependencies, add_paths_to_api, *rewrite, + *hyperapp, *reproducible, *force, *verbose, @@ -758,6 +762,13 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Rewrite the package (disables `Spawn!()`) [default: don't rewrite]") .required(false) ) + .arg(Arg::new("HYPERAPP") + .action(ArgAction::SetTrue) + .short('h') + .long("hyperapp") + .help("Build using the Hyperapp framework [default: don't use Hyperapp framework]") + .required(false) + ) .arg(Arg::new("REPRODUCIBLE") .action(ArgAction::SetTrue) .short('r') @@ -865,6 +876,13 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Rewrite the package (disables `Spawn!()`) [default: don't rewrite]") .required(false) ) + .arg(Arg::new("HYPERAPP") + .action(ArgAction::SetTrue) + .short('h') + .long("hyperapp") + .help("Build using the Hyperapp framework [default: don't use Hyperapp framework]") + .required(false) + ) .arg(Arg::new("REPRODUCIBLE") .action(ArgAction::SetTrue) .short('r') diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index fbc6afca..cbdb1afd 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -355,6 +355,7 @@ async fn build_packages( let url = format!("http://localhost:{port}"); + // TODO: add hyperapp setting to tests.toml for dependency_package_path in &test.dependency_package_paths { let path = match expand_home_path(&dependency_package_path) { Some(p) => p, @@ -381,6 +382,7 @@ async fn build_packages( false, false, false, + false, ) .await?; debug!("Start {path:?}"); @@ -406,6 +408,7 @@ async fn build_packages( false, false, false, + false, ) .await?; } @@ -428,6 +431,7 @@ async fn build_packages( false, false, false, + false, ) .await?; } From 04b4b72483598f0a531e793beeb7808a1cb52a9d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 4 Apr 2025 00:21:41 +0000 Subject: [PATCH 3/8] Format Rust code using rustfmt --- src/build/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index b8059fc7..509a784b 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1763,7 +1763,8 @@ pub async fn execute( if hyperapp { let api_dir = live_dir.join("api"); - let (_processed_projects, _interfaces) = wit_generator::generate_wit_files(&live_dir, &api_dir)?; + let (_processed_projects, _interfaces) = + wit_generator::generate_wit_files(&live_dir, &api_dir)?; } let ui_dirs = get_ui_dirs(&live_dir, &include, &exclude)?; From a4a7ea478ba51c3e16aa13f250fce05e601189e1 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 3 Apr 2025 17:27:50 -0700 Subject: [PATCH 4/8] build: remove `-h` flag that was interferring with `--help` --- src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 758cde2f..a49c9139 100644 --- a/src/main.rs +++ b/src/main.rs @@ -764,7 +764,6 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { ) .arg(Arg::new("HYPERAPP") .action(ArgAction::SetTrue) - .short('h') .long("hyperapp") .help("Build using the Hyperapp framework [default: don't use Hyperapp framework]") .required(false) @@ -878,7 +877,6 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { ) .arg(Arg::new("HYPERAPP") .action(ArgAction::SetTrue) - .short('h') .long("hyperapp") .help("Build using the Hyperapp framework [default: don't use Hyperapp framework]") .required(false) From 4ad1b14bd730c210040757e8eed4e25d70ba6955 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Mon, 7 Apr 2025 21:37:23 -0700 Subject: [PATCH 5/8] build: error out of bindgen rather than making non-working wit --- src/build/caller_utils_generator.rs | 433 ++++++++++++++++------------ src/build/wit_generator.rs | 81 +++--- 2 files changed, 295 insertions(+), 219 deletions(-) diff --git a/src/build/caller_utils_generator.rs b/src/build/caller_utils_generator.rs index 8de9c0f9..f6819a5e 100644 --- a/src/build/caller_utils_generator.rs +++ b/src/build/caller_utils_generator.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -14,7 +14,7 @@ pub fn to_snake_case(s: &str) -> String { pub fn to_pascal_case(s: &str) -> String { let parts = s.split('-'); let mut result = String::new(); - + for part in parts { if !part.is_empty() { let mut chars = part.chars(); @@ -24,7 +24,7 @@ pub fn to_pascal_case(s: &str) -> String { } } } - + result } @@ -32,7 +32,7 @@ pub fn to_pascal_case(s: &str) -> String { fn find_world_name(api_dir: &Path) -> Result { let mut regular_world_name = None; let mut types_world_name = None; - + // Look for world definition files for entry in WalkDir::new(api_dir) .max_depth(1) @@ -40,22 +40,24 @@ fn find_world_name(api_dir: &Path) -> Result { .filter_map(Result::ok) { let path = entry.path(); - + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { if let Ok(content) = fs::read_to_string(path) { if content.contains("world ") { println!("Analyzing world definition file: {}", path.display()); - + // Extract the world name let lines: Vec<&str> = content.lines().collect(); - - if let Some(world_line) = lines.iter().find(|line| line.trim().starts_with("world ")) { + + if let Some(world_line) = + lines.iter().find(|line| line.trim().starts_with("world ")) + { println!("World line: {}", world_line); - + if let Some(world_name) = world_line.trim().split_whitespace().nth(1) { let clean_name = world_name.trim_end_matches(" {"); println!("Extracted world name: {}", clean_name); - + // Check if this is a types-prefixed world if clean_name.starts_with("types-") { types_world_name = Some(clean_name.to_string()); @@ -70,28 +72,31 @@ fn find_world_name(api_dir: &Path) -> Result { } } } - + // Prioritize types-prefixed world if found if let Some(types_name) = types_world_name { return Ok(types_name); } - + // If no types-prefixed world found, check if we have a regular world if let Some(regular_name) = regular_world_name { // Check if there's a corresponding types-prefixed world file let types_name = format!("types-{}", regular_name); let types_file = api_dir.join(format!("{}.wit", types_name)); - + if types_file.exists() { println!("Found types world from file: {}", types_name); return Ok(types_name); } - + // Fall back to regular world but print a warning - println!("Warning: No types- world found, using regular world: {}", regular_name); + println!( + "Warning: No types- world found, using regular world: {}", + regular_name + ); return Ok(regular_name); } - + // If no world name is found, we should fail bail!("No world name found in any WIT file. Cannot generate caller-utils without a world name.") } @@ -108,9 +113,6 @@ fn wit_type_to_rust(wit_type: &str) -> String { "u32" => "u32".to_string(), "s64" => "i64".to_string(), "u64" => "u64".to_string(), - // Size types - "usize" => "usize".to_string(), - "isize" => "isize".to_string(), // Floating point types "f32" => "f32".to_string(), "f64" => "f64".to_string(), @@ -119,33 +121,32 @@ fn wit_type_to_rust(wit_type: &str) -> String { "str" => "&str".to_string(), "char" => "char".to_string(), "bool" => "bool".to_string(), - "unit" => "()".to_string(), + "_" => "()".to_string(), // Special types "address" => "WitAddress".to_string(), - // Common primitives that might be written differently in WIT - "i8" => "i8".to_string(), - "i16" => "i16".to_string(), - "i32" => "i32".to_string(), - "i64" => "i64".to_string(), // Collection types with generics t if t.starts_with("list<") => { let inner_type = &t[5..t.len() - 1]; format!("Vec<{}>", wit_type_to_rust(inner_type)) - }, + } t if t.starts_with("option<") => { let inner_type = &t[7..t.len() - 1]; format!("Option<{}>", wit_type_to_rust(inner_type)) - }, + } t if t.starts_with("result<") => { let inner_part = &t[7..t.len() - 1]; if let Some(comma_pos) = inner_part.find(',') { let ok_type = &inner_part[..comma_pos].trim(); let err_type = &inner_part[comma_pos + 1..].trim(); - format!("Result<{}, {}>", wit_type_to_rust(ok_type), wit_type_to_rust(err_type)) + format!( + "Result<{}, {}>", + wit_type_to_rust(ok_type), + wit_type_to_rust(err_type) + ) } else { format!("Result<{}, ()>", wit_type_to_rust(inner_part)) } - }, + } t if t.starts_with("tuple<") => { let inner_types = &t[6..t.len() - 1]; let rust_types: Vec = inner_types @@ -153,19 +154,7 @@ fn wit_type_to_rust(wit_type: &str) -> String { .map(|t| wit_type_to_rust(t)) .collect(); format!("({})", rust_types.join(", ")) - }, - // Handle map type if present - t if t.starts_with("map<") => { - let inner_part = &t[4..t.len() - 1]; - if let Some(comma_pos) = inner_part.find(',') { - let key_type = &inner_part[..comma_pos].trim(); - let value_type = &inner_part[comma_pos + 1..].trim(); - format!("HashMap<{}, {}>", wit_type_to_rust(key_type), wit_type_to_rust(value_type)) - } else { - // Fallback for malformed map type - format!("HashMap", wit_type_to_rust(inner_part)) - } - }, + } // Custom types (in kebab-case) need to be converted to PascalCase _ => to_pascal_case(wit_type).to_string(), } @@ -175,7 +164,9 @@ fn wit_type_to_rust(wit_type: &str) -> String { fn generate_default_value(rust_type: &str) -> String { match rust_type { // Integer types - "i8" | "u8" | "i16" | "u16" | "i32" | "u32" | "i64" | "u64" | "isize" | "usize" => "0".to_string(), + "i8" | "u8" | "i16" | "u16" | "i32" | "u32" | "i64" | "u64" | "isize" | "usize" => { + "0".to_string() + } // Floating point types "f32" | "f64" => "0.0".to_string(), // String types @@ -196,17 +187,18 @@ fn generate_default_value(rust_type: &str) -> String { } else { "Ok(())".to_string() } - }, - t if t.starts_with("HashMap<") => "HashMap::new()".to_string(), + } + //t if t.starts_with("HashMap<") => "HashMap::new()".to_string(), t if t.starts_with("(") => { // Generate default tuple with default values for each element let inner_part = t.trim_start_matches('(').trim_end_matches(')'); let parts: Vec<_> = inner_part.split(", ").collect(); - let default_values: Vec<_> = parts.iter() + let default_values: Vec<_> = parts + .iter() .map(|part| generate_default_value(part)) .collect(); format!("({})", default_values.join(", ")) - }, + } // For custom types, assume they implement Default _ => format!("{}::default()", rust_type), } @@ -228,7 +220,7 @@ struct SignatureStruct { // Find all interface imports in the world WIT file fn find_interfaces_in_world(api_dir: &Path) -> Result> { let mut interfaces = Vec::new(); - + // Find world definition files for entry in WalkDir::new(api_dir) .max_depth(1) @@ -236,12 +228,12 @@ fn find_interfaces_in_world(api_dir: &Path) -> Result> { .filter_map(Result::ok) { let path = entry.path(); - + if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { if let Ok(content) = fs::read_to_string(path) { if content.contains("world ") { println!("Analyzing world definition file: {}", path.display()); - + // Extract import statements for line in content.lines() { let line = line.trim(); @@ -250,7 +242,7 @@ fn find_interfaces_in_world(api_dir: &Path) -> Result> { .trim_start_matches("import ") .trim_end_matches(";") .trim(); - + interfaces.push(interface.to_string()); println!(" Found interface import: {}", interface); } @@ -259,44 +251,53 @@ fn find_interfaces_in_world(api_dir: &Path) -> Result> { } } } - + Ok(interfaces) } // Parse WIT file to extract function signatures and type definitions fn parse_wit_file(file_path: &Path) -> Result<(Vec, Vec)> { println!("Parsing WIT file: {}", file_path.display()); - + let content = fs::read_to_string(file_path) .with_context(|| format!("Failed to read WIT file: {}", file_path.display()))?; - + let mut signatures = Vec::new(); let mut type_names = Vec::new(); - + // Simple parser for WIT files to extract record definitions and types let lines: Vec<_> = content.lines().collect(); let mut i = 0; - + while i < lines.len() { let line = lines[i].trim(); - + // Look for record definitions that aren't signature structs if line.starts_with("record ") && !line.contains("-signature-") { - let record_name = line.trim_start_matches("record ").trim_end_matches(" {").trim(); + let record_name = line + .trim_start_matches("record ") + .trim_end_matches(" {") + .trim(); println!(" Found type: record {}", record_name); type_names.push(record_name.to_string()); } // Look for variant definitions (enums) else if line.starts_with("variant ") { - let variant_name = line.trim_start_matches("variant ").trim_end_matches(" {").trim(); + let variant_name = line + .trim_start_matches("variant ") + .trim_end_matches(" {") + .trim(); println!(" Found type: variant {}", variant_name); type_names.push(variant_name.to_string()); } // Look for signature record definitions else if line.starts_with("record ") && line.contains("-signature-") { - let record_name = line.trim_start_matches("record ").trim_end_matches(" {").trim(); + let record_name = line + .trim_start_matches("record ") + .trim_end_matches(" {") + .trim(); println!(" Found record: {}", record_name); - + // Extract function name and attribute type let parts: Vec<_> = record_name.split("-signature-").collect(); if parts.len() != 2 { @@ -304,51 +305,55 @@ fn parse_wit_file(file_path: &Path) -> Result<(Vec, Vec i += 1; continue; } - + let function_name = parts[0].to_string(); let attr_type = parts[1].to_string(); - + // Parse fields let mut fields = Vec::new(); i += 1; - + while i < lines.len() && !lines[i].trim().starts_with("}") { let field_line = lines[i].trim(); - + // Skip comments and empty lines if field_line.starts_with("//") || field_line.is_empty() { i += 1; continue; } - + // Parse field definition let field_parts: Vec<_> = field_line.split(':').collect(); if field_parts.len() == 2 { let field_name = field_parts[0].trim().to_string(); let field_type = field_parts[1].trim().trim_end_matches(',').to_string(); - + println!(" Field: {} -> {}", field_name, field_type); fields.push(SignatureField { name: field_name, wit_type: field_type, }); } - + i += 1; } - + signatures.push(SignatureStruct { function_name, attr_type, fields, }); } - + i += 1; } - - println!("Extracted {} signature structs and {} type definitions from {}", - signatures.len(), type_names.len(), file_path.display()); + + println!( + "Extracted {} signature structs and {} type definitions from {}", + signatures.len(), + type_names.len(), + file_path.display() + ); Ok((signatures, type_names)) } @@ -356,23 +361,23 @@ fn parse_wit_file(file_path: &Path) -> Result<(Vec, Vec fn generate_async_function(signature: &SignatureStruct) -> String { // Convert function name from kebab-case to snake_case let snake_function_name = to_snake_case(&signature.function_name); - + // Get pascal case version for the JSON request format let pascal_function_name = to_pascal_case(&signature.function_name); - + // Function full name with attribute type let full_function_name = format!("{}_{}_rpc", snake_function_name, signature.attr_type); - + // Extract parameters and return type let mut params = Vec::new(); let mut param_names = Vec::new(); let mut return_type = "()".to_string(); let mut target_param = ""; - + for field in &signature.fields { let field_name_snake = to_snake_case(&field.name); let rust_type = wit_type_to_rust(&field.wit_type); - + if field.name == "target" { if field.wit_type == "string" { target_param = "&str"; @@ -387,24 +392,29 @@ fn generate_async_function(signature: &SignatureStruct) -> String { param_names.push(field_name_snake); } } - + // First parameter is always target let all_params = if target_param.is_empty() { params.join(", ") } else { - format!("target: {}{}", target_param, if params.is_empty() { "" } else { ", " }) + ¶ms.join(", ") + format!( + "target: {}{}", + target_param, + if params.is_empty() { "" } else { ", " } + ) + ¶ms.join(", ") }; - + // Wrap the return type in SendResult let wrapped_return_type = format!("SendResult<{}>", return_type); - + // For HTTP endpoints, generate commented-out implementation if signature.attr_type == "http" { let default_value = generate_default_value(&return_type); - + // Add underscore prefix to all parameters for HTTP stubs let all_params_with_underscore = if target_param.is_empty() { - params.iter() + params + .iter() .map(|param| { let parts: Vec<&str> = param.split(':').collect(); if parts.len() == 2 { @@ -420,7 +430,8 @@ fn generate_async_function(signature: &SignatureStruct) -> String { if params.is_empty() { target_with_underscore } else { - let params_with_underscore = params.iter() + let params_with_underscore = params + .iter() .map(|param| { let parts: Vec<&str> = param.split(':').collect(); if parts.len() == 2 { @@ -434,7 +445,7 @@ fn generate_async_function(signature: &SignatureStruct) -> String { format!("{}, {}", target_with_underscore, params_with_underscore) } }; - + return format!( "/// Generated stub for `{}` {} RPC call\n/// HTTP endpoint - uncomment to implement\n// pub async fn {}({}) -> {} {{\n// // TODO: Implement HTTP endpoint\n// SendResult::Success({})\n// }}", signature.function_name, @@ -445,21 +456,26 @@ fn generate_async_function(signature: &SignatureStruct) -> String { default_value ); } - + // Format JSON parameters correctly let json_params = if param_names.is_empty() { // No parameters case format!("json!({{\"{}\" : {{}}}})", pascal_function_name) } else if param_names.len() == 1 { // Single parameter case - format!("json!({{\"{}\": {}}})", pascal_function_name, param_names[0]) + format!( + "json!({{\"{}\": {}}})", + pascal_function_name, param_names[0] + ) } else { // Multiple parameters case - use tuple format - format!("json!({{\"{}\": ({})}})", - pascal_function_name, - param_names.join(", ")) + format!( + "json!({{\"{}\": ({})}})", + pascal_function_name, + param_names.join(", ") + ) }; - + // Generate function with implementation using send format!( "/// Generated stub for `{}` {} RPC call\npub async fn {}({}) -> {} {{\n let request = {};\n send::<{}>(&request, target, 30).await\n}}", @@ -477,13 +493,16 @@ fn generate_async_function(signature: &SignatureStruct) -> String { fn create_caller_utils_crate(api_dir: &Path, base_dir: &Path) -> Result<()> { // Path to the new crate let caller_utils_dir = base_dir.join("caller-utils"); - println!("Creating caller-utils crate at {}", caller_utils_dir.display()); - + println!( + "Creating caller-utils crate at {}", + caller_utils_dir.display() + ); + // Create directories fs::create_dir_all(&caller_utils_dir)?; fs::create_dir_all(caller_utils_dir.join("src"))?; println!("Created project directory structure"); - + // Create Cargo.toml with updated dependencies let cargo_toml = r#"[package] name = "caller-utils" @@ -507,22 +526,22 @@ wit-bindgen = "0.41.0" [lib] crate-type = ["cdylib", "lib"] "#; - + fs::write(caller_utils_dir.join("Cargo.toml"), cargo_toml) .with_context(|| "Failed to write caller-utils Cargo.toml")?; - + println!("Created Cargo.toml for caller-utils"); - + // Get the world name (preferably the types- version) let world_name = find_world_name(api_dir)?; println!("Using world name for code generation: {}", world_name); - + // Get all interfaces from the world file let interface_imports = find_interfaces_in_world(api_dir)?; - + // Store all types from each interface let mut interface_types: HashMap> = HashMap::new(); - + // Find all WIT files in the api directory to generate stubs let mut wit_files = Vec::new(); for entry in WalkDir::new(api_dir) @@ -540,72 +559,79 @@ crate-type = ["cdylib", "lib"] } } } - + println!("Found {} WIT interface files", wit_files.len()); - + // Generate content for each module and collect types let mut module_contents = HashMap::::new(); - + for wit_file in &wit_files { // Extract the interface name from the file name let interface_name = wit_file.file_stem().unwrap().to_string_lossy(); let snake_interface_name = to_snake_case(&interface_name); - - println!("Processing interface: {} -> {}", interface_name, snake_interface_name); - + + println!( + "Processing interface: {} -> {}", + interface_name, snake_interface_name + ); + // Parse the WIT file to extract signature structs and types match parse_wit_file(wit_file) { Ok((signatures, types)) => { // Store types for this interface interface_types.insert(interface_name.to_string(), types); - + if signatures.is_empty() { println!("No signatures found in {}", wit_file.display()); continue; } - + // Generate module content let mut mod_content = String::new(); - + // Add function implementations for signature in &signatures { let function_impl = generate_async_function(signature); mod_content.push_str(&function_impl); mod_content.push_str("\n\n"); } - + // Store the module content module_contents.insert(snake_interface_name, mod_content); - - println!("Generated module content with {} function stubs", signatures.len()); - }, + + println!( + "Generated module content with {} function stubs", + signatures.len() + ); + } Err(e) => { println!("Error parsing WIT file {}: {}", wit_file.display(), e); } } } - + // Create import statements for each interface using "hyperware::process::{interface_name}::*" // Use a HashSet to track which interfaces we've already processed to avoid duplicates let mut processed_interfaces = std::collections::HashSet::new(); let mut interface_use_statements = Vec::new(); - + for interface_name in &interface_imports { // Convert to snake case for module name let snake_interface_name = to_snake_case(interface_name); - + // Only add the import if we haven't processed this interface yet if processed_interfaces.insert(snake_interface_name.clone()) { // Create wildcard import for this interface - interface_use_statements.push( - format!("pub use crate::hyperware::process::{}::*;", snake_interface_name) - ); + interface_use_statements.push(format!( + "pub use crate::hyperware::process::{}::*;", + snake_interface_name + )); } } - + // Create single lib.rs with all modules inline let mut lib_rs = String::new(); - + // Updated wit_bindgen usage with explicit world name - FIXED: Removed unused imports lib_rs.push_str("wit_bindgen::generate!({\n"); lib_rs.push_str(" path: \"target/wit\",\n"); @@ -613,15 +639,15 @@ crate-type = ["cdylib", "lib"] lib_rs.push_str(" generate_unused_types: true,\n"); lib_rs.push_str(" additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto],\n"); lib_rs.push_str("});\n\n"); - + lib_rs.push_str("/// Generated caller utilities for RPC function stubs\n\n"); - + // Add global imports lib_rs.push_str("pub use hyperware_app_common::SendResult;\n"); lib_rs.push_str("pub use hyperware_app_common::send;\n"); lib_rs.push_str("use hyperware_process_lib::Address;\n"); lib_rs.push_str("use serde_json::json;\n\n"); - + // Add interface use statements if !interface_use_statements.is_empty() { lib_rs.push_str("// Import types from each interface\n"); @@ -630,37 +656,40 @@ crate-type = ["cdylib", "lib"] } lib_rs.push_str("\n"); } - + // Add all modules with their content for (module_name, module_content) in module_contents { - lib_rs.push_str(&format!("/// Generated RPC stubs for the {} interface\n", module_name)); + lib_rs.push_str(&format!( + "/// Generated RPC stubs for the {} interface\n", + module_name + )); lib_rs.push_str(&format!("pub mod {} {{\n", module_name)); lib_rs.push_str(" use crate::*;\n\n"); lib_rs.push_str(&format!(" {}\n", module_content.replace("\n", "\n "))); lib_rs.push_str("}\n\n"); } - + // Write lib.rs let lib_rs_path = caller_utils_dir.join("src").join("lib.rs"); println!("Writing lib.rs to {}", lib_rs_path.display()); - + fs::write(&lib_rs_path, lib_rs) .with_context(|| format!("Failed to write lib.rs: {}", lib_rs_path.display()))?; - + println!("Created single lib.rs file with all modules inline"); - + // Create target/wit directory and copy all WIT files let target_wit_dir = caller_utils_dir.join("target").join("wit"); println!("Creating directory: {}", target_wit_dir.display()); - + // Remove the directory if it exists to ensure clean state if target_wit_dir.exists() { println!("Removing existing target/wit directory"); fs::remove_dir_all(&target_wit_dir)?; } - + fs::create_dir_all(&target_wit_dir)?; - + // Copy all WIT files to target/wit for entry in WalkDir::new(api_dir) .max_depth(1) @@ -671,52 +700,75 @@ crate-type = ["cdylib", "lib"] if path.is_file() && path.extension().map_or(false, |ext| ext == "wit") { let file_name = path.file_name().unwrap(); let target_path = target_wit_dir.join(file_name); - fs::copy(path, &target_path) - .with_context(|| format!("Failed to copy {} to {}", path.display(), target_path.display()))?; - println!("Copied {} to target/wit directory", file_name.to_string_lossy()); + fs::copy(path, &target_path).with_context(|| { + format!( + "Failed to copy {} to {}", + path.display(), + target_path.display() + ) + })?; + println!( + "Copied {} to target/wit directory", + file_name.to_string_lossy() + ); } } - + Ok(()) } // Update workspace Cargo.toml to include the caller-utils crate fn update_workspace_cargo_toml(base_dir: &Path) -> Result<()> { let workspace_cargo_toml = base_dir.join("Cargo.toml"); - println!("Updating workspace Cargo.toml at {}", workspace_cargo_toml.display()); - + println!( + "Updating workspace Cargo.toml at {}", + workspace_cargo_toml.display() + ); + if !workspace_cargo_toml.exists() { - println!("Workspace Cargo.toml not found at {}", workspace_cargo_toml.display()); + println!( + "Workspace Cargo.toml not found at {}", + workspace_cargo_toml.display() + ); return Ok(()); } - - let content = fs::read_to_string(&workspace_cargo_toml) - .with_context(|| format!("Failed to read workspace Cargo.toml: {}", workspace_cargo_toml.display()))?; - + + let content = fs::read_to_string(&workspace_cargo_toml).with_context(|| { + format!( + "Failed to read workspace Cargo.toml: {}", + workspace_cargo_toml.display() + ) + })?; + // Parse the TOML content - let mut parsed_toml: Value = content.parse() + let mut parsed_toml: Value = content + .parse() .with_context(|| "Failed to parse workspace Cargo.toml")?; - + // Check if there's a workspace section if let Some(workspace) = parsed_toml.get_mut("workspace") { if let Some(members) = workspace.get_mut("members") { if let Some(members_array) = members.as_array_mut() { // Check if caller-utils is already in the members list - let caller_utils_exists = members_array.iter().any(|m| { - m.as_str().map_or(false, |s| s == "caller-utils") - }); - + let caller_utils_exists = members_array + .iter() + .any(|m| m.as_str().map_or(false, |s| s == "caller-utils")); + if !caller_utils_exists { println!("Adding caller-utils to workspace members"); members_array.push(Value::String("caller-utils".to_string())); - + // Write back the updated TOML let updated_content = toml::to_string_pretty(&parsed_toml) .with_context(|| "Failed to serialize updated workspace Cargo.toml")?; - - fs::write(&workspace_cargo_toml, updated_content) - .with_context(|| format!("Failed to write updated workspace Cargo.toml: {}", workspace_cargo_toml.display()))?; - + + fs::write(&workspace_cargo_toml, updated_content).with_context(|| { + format!( + "Failed to write updated workspace Cargo.toml: {}", + workspace_cargo_toml.display() + ) + })?; + println!("Successfully updated workspace Cargo.toml"); } else { println!("caller-utils is already in workspace members"); @@ -724,7 +776,7 @@ fn update_workspace_cargo_toml(base_dir: &Path) -> Result<()> { } } } - + Ok(()) } @@ -732,14 +784,25 @@ fn update_workspace_cargo_toml(base_dir: &Path) -> Result<()> { fn add_caller_utils_to_projects(projects: &[PathBuf]) -> Result<()> { for project_path in projects { let cargo_toml_path = project_path.join("Cargo.toml"); - println!("Adding caller-utils dependency to {}", cargo_toml_path.display()); - - let content = fs::read_to_string(&cargo_toml_path) - .with_context(|| format!("Failed to read project Cargo.toml: {}", cargo_toml_path.display()))?; - - let mut parsed_toml: Value = content.parse() - .with_context(|| format!("Failed to parse project Cargo.toml: {}", cargo_toml_path.display()))?; - + println!( + "Adding caller-utils dependency to {}", + cargo_toml_path.display() + ); + + let content = fs::read_to_string(&cargo_toml_path).with_context(|| { + format!( + "Failed to read project Cargo.toml: {}", + cargo_toml_path.display() + ) + })?; + + let mut parsed_toml: Value = content.parse().with_context(|| { + format!( + "Failed to parse project Cargo.toml: {}", + cargo_toml_path.display() + ) + })?; + // Add caller-utils to dependencies if not already present if let Some(dependencies) = parsed_toml.get_mut("dependencies") { if let Some(deps_table) = dependencies.as_table_mut() { @@ -748,18 +811,30 @@ fn add_caller_utils_to_projects(projects: &[PathBuf]) -> Result<()> { "caller-utils".to_string(), Value::Table({ let mut t = toml::map::Map::new(); - t.insert("path".to_string(), Value::String("../caller-utils".to_string())); + t.insert( + "path".to_string(), + Value::String("../caller-utils".to_string()), + ); t - }) + }), ); - + // Write back the updated TOML - let updated_content = toml::to_string_pretty(&parsed_toml) - .with_context(|| format!("Failed to serialize updated project Cargo.toml: {}", cargo_toml_path.display()))?; - - fs::write(&cargo_toml_path, updated_content) - .with_context(|| format!("Failed to write updated project Cargo.toml: {}", cargo_toml_path.display()))?; - + let updated_content = + toml::to_string_pretty(&parsed_toml).with_context(|| { + format!( + "Failed to serialize updated project Cargo.toml: {}", + cargo_toml_path.display() + ) + })?; + + fs::write(&cargo_toml_path, updated_content).with_context(|| { + format!( + "Failed to write updated project Cargo.toml: {}", + cargo_toml_path.display() + ) + })?; + println!("Successfully added caller-utils dependency"); } else { println!("caller-utils dependency already exists"); @@ -767,7 +842,7 @@ fn add_caller_utils_to_projects(projects: &[PathBuf]) -> Result<()> { } } } - + Ok(()) } @@ -775,12 +850,12 @@ fn add_caller_utils_to_projects(projects: &[PathBuf]) -> Result<()> { pub fn create_caller_utils(base_dir: &Path, api_dir: &Path, projects: &[PathBuf]) -> Result<()> { // Step 1: Create the caller-utils crate create_caller_utils_crate(api_dir, base_dir)?; - + // Step 2: Update workspace Cargo.toml update_workspace_cargo_toml(base_dir)?; - + // Step 3: Add caller-utils dependency to each hyperware:process project add_caller_utils_to_projects(projects)?; - + Ok(()) -} \ No newline at end of file +} diff --git a/src/build/wit_generator.rs b/src/build/wit_generator.rs index 43fffb6e..4bf22a91 100644 --- a/src/build/wit_generator.rs +++ b/src/build/wit_generator.rs @@ -1,10 +1,9 @@ -//use anyhow::{Context}; use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use color_eyre::{ - eyre::{bail, WrapErr}, + eyre::{bail, eyre, WrapErr}, Result, }; use syn::{self, Attribute, ImplItem, Item, Type}; @@ -109,13 +108,17 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result { if type_path.path.segments.is_empty() { - return Ok("unknown".to_string()); + return Err(eyre!("Failed to parse path type: {ty:?}")); } let ident = &type_path.path.segments.last().unwrap().ident; let type_name = ident.to_string(); match type_name.as_str() { + "i8" => Ok("s8".to_string()), + "u8" => Ok("u8".to_string()), + "i16" => Ok("s16".to_string()), + "u16" => Ok("u16".to_string()), "i32" => Ok("s32".to_string()), "u32" => Ok("u32".to_string()), "i64" => Ok("s64".to_string()), @@ -132,10 +135,10 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result", inner_type)) } else { - Ok("list".to_string()) + Err(eyre!("Failed to parse Vec inner type")) } } else { - Ok("list".to_string()) + Err(eyre!("Failed to parse Vec inner type!")) } } "Option" => { @@ -146,36 +149,37 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result", inner_type)) } else { - Ok("option".to_string()) + Err(eyre!("Failed to parse Option inner type")) } } else { - Ok("option".to_string()) - } - } - "HashMap" | "BTreeMap" => { - if let syn::PathArguments::AngleBracketed(args) = - &type_path.path.segments.last().unwrap().arguments - { - if args.args.len() >= 2 { - if let ( - Some(syn::GenericArgument::Type(key_ty)), - Some(syn::GenericArgument::Type(val_ty)), - ) = (args.args.first(), args.args.get(1)) - { - let key_type = rust_type_to_wit(key_ty, used_types)?; - let val_type = rust_type_to_wit(val_ty, used_types)?; - // For HashMaps, we'll generate a list of tuples where each tuple contains a key and value - Ok(format!("list>", key_type, val_type)) - } else { - Ok("list>".to_string()) - } - } else { - Ok("list>".to_string()) - } - } else { - Ok("list>".to_string()) + Err(eyre!("Failed to parse Option inner type!")) } } + // TODO: fix and enable + //"HashMap" | "BTreeMap" => { + // if let syn::PathArguments::AngleBracketed(args) = + // &type_path.path.segments.last().unwrap().arguments + // { + // if args.args.len() >= 2 { + // if let ( + // Some(syn::GenericArgument::Type(key_ty)), + // Some(syn::GenericArgument::Type(val_ty)), + // ) = (args.args.first(), args.args.get(1)) + // { + // let key_type = rust_type_to_wit(key_ty, used_types)?; + // let val_type = rust_type_to_wit(val_ty, used_types)?; + // // For HashMaps, we'll generate a list of tuples where each tuple contains a key and value + // Ok(format!("list>", key_type, val_type)) + // } else { + // Ok("list>".to_string()) + // } + // } else { + // Ok("list>".to_string()) + // } + // } else { + // Ok("list>".to_string()) + // } + //} custom => { // Validate custom type name validate_name(custom, "Type")?; @@ -204,7 +208,7 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result", elem_types.join(", "))) } } - _ => Ok("unknown".to_string()), + _ => return Err(eyre!("Failed to parse type: {ty:?}")), } } @@ -296,7 +300,7 @@ fn collect_type_definitions_from_file(file_path: &Path) -> Result Result { println!(" Error converting parameter type: {}", e); - // Use a placeholder type for this parameter - struct_fields.push(format!(" {}: unknown", param_name)); + return Err(e); } } } Err(e) => { println!(" Skipping parameter with invalid name: {}", e); - // Use a placeholder for invalid parameter names - struct_fields.push(" invalid-param: unknown".to_string()); + return Err(e); } } } @@ -577,7 +578,7 @@ fn generate_signature_struct( } Err(e) => { println!(" Error converting return type: {}", e); - struct_fields.push(" returning: unknown".to_string()); + return Err(e); } }, _ => { From a0c9aeb4c14a3d8c2288e9093979b3e7ad43a5e8 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 8 Apr 2025 05:05:54 -0700 Subject: [PATCH 6/8] build: if no hyperapp api dir, make it --- src/build/mod.rs | 2 +- src/build/wit_generator.rs | 232 ++++++++++++++++++++++--------------- 2 files changed, 141 insertions(+), 93 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 509a784b..577c5892 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1764,7 +1764,7 @@ pub async fn execute( if hyperapp { let api_dir = live_dir.join("api"); let (_processed_projects, _interfaces) = - wit_generator::generate_wit_files(&live_dir, &api_dir)?; + wit_generator::generate_wit_files(&live_dir, &api_dir, false)?; } let ui_dirs = get_ui_dirs(&live_dir, &include, &exclude)?; diff --git a/src/build/wit_generator.rs b/src/build/wit_generator.rs index 4bf22a91..4971792a 100644 --- a/src/build/wit_generator.rs +++ b/src/build/wit_generator.rs @@ -613,7 +613,7 @@ impl AsTypePath for syn::Type { } // Process a single Rust project and generate WIT files -fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result> { +fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result> { println!("\nProcessing project: {}", project_path.display()); // Find lib.rs for this project @@ -864,60 +864,24 @@ fn process_rust_project(project_path: &Path, api_dir: &Path) -> Result Result<(Vec, Vec)> { - // Find all relevant Rust projects - let projects = find_rust_projects(base_dir); - let mut processed_projects = Vec::new(); - - if projects.is_empty() { - println!("No relevant Rust projects found."); - return Ok((Vec::new(), Vec::new())); - } - - // Process each project and collect world imports - let mut new_imports = Vec::new(); - let mut interfaces = Vec::new(); - - for project_path in &projects { - println!("Processing project: {}", project_path.display()); - - match process_rust_project(project_path, api_dir) { - Ok(Some(import)) => { - println!("Got import statement: {}", import); - new_imports.push(import.clone()); - - // Extract interface name from import statement - let interface_name = import - .trim_start_matches(" import ") - .trim_end_matches(";") - .to_string(); - - interfaces.push(interface_name); - processed_projects.push(project_path.clone()); - } - Ok(None) => println!("No import statement generated"), - Err(e) => println!("Error processing project: {}", e), - } - } - - println!("Collected {} new imports", new_imports.len()); - - // Check for existing world definition files and update them - println!("Looking for existing world definition files"); - let mut updated_world = false; - +fn rewrite_wit( + api_dir: &Path, + new_imports: &Vec, + wit_worlds: &mut HashSet, + updated_world: &mut bool, +) -> Result<()> { for entry in WalkDir::new(api_dir) .max_depth(1) .into_iter() @@ -955,62 +919,146 @@ pub fn generate_wit_files(base_dir: &Path, api_dir: &Path) -> Result<(Vec = all_imports - .iter() - .map(|import| { - if import.starts_with(" ") { - import.clone() - } else { - format!(" {}", import.trim()) - } - }) - .collect(); - - let imports_section = all_imports_with_indent.join("\n"); - - // Create updated world content with proper indentation - let world_content = format!( - "world {} {{\n{}\n {}\n}}", - world_name, - imports_section, - include_line.trim() - ); - - println!("Writing updated world definition to {}", path.display()); - // Write the updated world file - fs::write(path, world_content).with_context(|| { - format!("Failed to write updated world file: {}", path.display()) - })?; - - println!("Successfully updated world definition"); - updated_world = true; + // Make sure all imports have proper indentation + let all_imports_with_indent: Vec = all_imports + .iter() + .map(|import| { + if import.starts_with(" ") { + import.clone() + } else { + format!(" {}", import.trim()) + } + }) + .collect(); + + let imports_section = all_imports_with_indent.join("\n"); + + // Create updated world content with proper indentation + let world_content = format!( + "world {} {{\n{}\n {}\n}}", + world_name, + imports_section, + include_line.trim() + ); + + println!("Writing updated world definition to {}", path.display()); + // Write the updated world file + fs::write(path, world_content).with_context(|| { + format!("Failed to write updated world file: {}", path.display()) + })?; + + println!("Successfully updated world definition"); + *updated_world = true; + } } } } } } + Ok(()) +} + +// Generate WIT files from Rust code +pub fn generate_wit_files( + base_dir: &Path, + api_dir: &Path, + is_recursive_call: bool, +) -> Result<(Vec, Vec)> { + fs::create_dir_all(&api_dir)?; + + // Find all relevant Rust projects + let projects = find_rust_projects(base_dir); + let mut processed_projects = Vec::new(); + + if projects.is_empty() { + println!("No relevant Rust projects found."); + return Ok((Vec::new(), Vec::new())); + } + + // Process each project and collect world imports + let mut new_imports = Vec::new(); + let mut interfaces = Vec::new(); + + let mut wit_worlds = HashSet::new(); + for project_path in &projects { + println!("Processing project: {}", project_path.display()); + + match process_rust_project(project_path, api_dir) { + Ok(Some((import, wit_world))) => { + println!("Got import statement: {}", import); + new_imports.push(import.clone()); + + // Extract interface name from import statement + let interface_name = import + .trim_start_matches(" import ") + .trim_end_matches(";") + .to_string(); + + interfaces.push(interface_name); + processed_projects.push(project_path.clone()); + + wit_worlds.insert(wit_world); + } + Ok(None) => println!("No import statement generated"), + Err(e) => println!("Error processing project: {}", e), + } + } + + println!("Collected {} new imports", new_imports.len()); + + // Check for existing world definition files and update them + println!("Looking for existing world definition files"); + let mut updated_world = false; + + rewrite_wit(api_dir, &new_imports, &mut wit_worlds, &mut updated_world)?; + + let rerun_rewrite_wit = !wit_worlds.is_empty(); + for wit_world in wit_worlds { + // Create a new file with the simple world definition + let new_file_path = api_dir.join(format!("{}.wit", wit_world)); + let simple_world_content = format!("world {} {{}}", wit_world); + + println!( + "Creating new world definition file: {}", + new_file_path.display() + ); + fs::write(&new_file_path, simple_world_content).with_context(|| { + format!( + "Failed to create new world file: {}", + new_file_path.display() + ) + })?; + + println!("Successfully created new world definition file"); + updated_world = true; + } + + if rerun_rewrite_wit && !is_recursive_call { + return generate_wit_files(base_dir, api_dir, true); + } // If no world definitions were found, create a default one if !updated_world && !new_imports.is_empty() { From 78a62c8a00715a8f4930cf9d3c8ef5355ca3fb9f Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 8 Apr 2025 20:22:27 -0700 Subject: [PATCH 7/8] build: get caller_utils working for deps --- src/build/caller_utils_generator.rs | 82 ++++++++++++-------------- src/build/mod.rs | 90 ++++++++++++++++++++--------- src/build/wit_generator.rs | 39 ++++++++++++- 3 files changed, 139 insertions(+), 72 deletions(-) diff --git a/src/build/caller_utils_generator.rs b/src/build/caller_utils_generator.rs index f6819a5e..aaeed467 100644 --- a/src/build/caller_utils_generator.rs +++ b/src/build/caller_utils_generator.rs @@ -1,7 +1,12 @@ -use anyhow::{bail, Context, Result}; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; + +use color_eyre::{ + eyre::{bail, WrapErr}, + Result, +}; + use toml::Value; use walkdir::WalkDir; @@ -29,9 +34,8 @@ pub fn to_pascal_case(s: &str) -> String { } // Find the world name in the world WIT file, prioritizing types-prefixed worlds -fn find_world_name(api_dir: &Path) -> Result { - let mut regular_world_name = None; - let mut types_world_name = None; +fn find_world_names(api_dir: &Path) -> Result> { + let mut world_names = Vec::new(); // Look for world definition files for entry in WalkDir::new(api_dir) @@ -60,11 +64,8 @@ fn find_world_name(api_dir: &Path) -> Result { // Check if this is a types-prefixed world if clean_name.starts_with("types-") { - types_world_name = Some(clean_name.to_string()); + world_names.push(clean_name.to_string()); println!("Found types world: {}", clean_name); - } else { - regular_world_name = Some(clean_name.to_string()); - println!("Found regular world: {}", clean_name); } } } @@ -73,32 +74,10 @@ fn find_world_name(api_dir: &Path) -> Result { } } - // Prioritize types-prefixed world if found - if let Some(types_name) = types_world_name { - return Ok(types_name); + if world_names.is_empty() { + bail!("No world name found in any WIT file. Cannot generate caller-utils without a world name.") } - - // If no types-prefixed world found, check if we have a regular world - if let Some(regular_name) = regular_world_name { - // Check if there's a corresponding types-prefixed world file - let types_name = format!("types-{}", regular_name); - let types_file = api_dir.join(format!("{}.wit", types_name)); - - if types_file.exists() { - println!("Found types world from file: {}", types_name); - return Ok(types_name); - } - - // Fall back to regular world but print a warning - println!( - "Warning: No types- world found, using regular world: {}", - regular_name - ); - return Ok(regular_name); - } - - // If no world name is found, we should fail - bail!("No world name found in any WIT file. Cannot generate caller-utils without a world name.") + Ok(world_names) } // Convert WIT type to Rust type - IMPROVED with more Rust primitives @@ -447,7 +426,7 @@ fn generate_async_function(signature: &SignatureStruct) -> String { }; return format!( - "/// Generated stub for `{}` {} RPC call\n/// HTTP endpoint - uncomment to implement\n// pub async fn {}({}) -> {} {{\n// // TODO: Implement HTTP endpoint\n// SendResult::Success({})\n// }}", + "// /// Generated stub for `{}` {} RPC call\n// /// HTTP endpoint - uncomment to implement\n// pub async fn {}({}) -> {} {{\n// // TODO: Implement HTTP endpoint\n// SendResult::Success({})\n// }}", signature.function_name, signature.attr_type, full_function_name, @@ -478,7 +457,7 @@ fn generate_async_function(signature: &SignatureStruct) -> String { // Generate function with implementation using send format!( - "/// Generated stub for `{}` {} RPC call\npub async fn {}({}) -> {} {{\n let request = {};\n send::<{}>(&request, target, 30).await\n}}", + "/// Generated stub for `{}` {} RPC call\npub async fn {}({}) -> {} {{\n let body = {};\n let body = serde_json::to_vec(&body).unwrap();\n let request = Request::to(target)\n .body(body);\n send::<{}>(request).await\n}}", signature.function_name, signature.attr_type, full_function_name, @@ -492,7 +471,7 @@ fn generate_async_function(signature: &SignatureStruct) -> String { // Create the caller-utils crate with a single lib.rs file fn create_caller_utils_crate(api_dir: &Path, base_dir: &Path) -> Result<()> { // Path to the new crate - let caller_utils_dir = base_dir.join("caller-utils"); + let caller_utils_dir = base_dir.join("crates").join("caller-utils"); println!( "Creating caller-utils crate at {}", caller_utils_dir.display() @@ -512,12 +491,11 @@ publish = false [dependencies] anyhow = "1.0" -hyperware_process_lib = { version = "1.0.4", features = ["logging"] } process_macros = "0.1.0" futures-util = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -hyperware_app_common = { git = "https://github.com/hyperware-ai/hyperprocess-macro" } +hyperware_app_common = { git = "https://github.com/hyperware-ai/hyperprocess-macro", rev = "4e417e1" } once_cell = "1.20.2" futures = "0.3" uuid = { version = "1.0" } @@ -533,8 +511,22 @@ crate-type = ["cdylib", "lib"] println!("Created Cargo.toml for caller-utils"); // Get the world name (preferably the types- version) - let world_name = find_world_name(api_dir)?; - println!("Using world name for code generation: {}", world_name); + let world_names = find_world_names(api_dir)?; + println!("Using world names for code generation: {:?}", world_names); + let world_name = if world_names.len() == 0 { + "" + } else if world_names.len() == 1 { + &world_names[0] + } else { + let path = api_dir.join("types.wit"); + let mut content = "world types {\n".to_string(); + for world_name in world_names { + content.push_str(&format!(" include {world_name};\n")); + } + content.push_str("}\n"); + fs::write(&path, &content)?; + "types" + }; // Get all interfaces from the world file let interface_imports = find_interfaces_in_world(api_dir)?; @@ -632,7 +624,6 @@ crate-type = ["cdylib", "lib"] // Create single lib.rs with all modules inline let mut lib_rs = String::new(); - // Updated wit_bindgen usage with explicit world name - FIXED: Removed unused imports lib_rs.push_str("wit_bindgen::generate!({\n"); lib_rs.push_str(" path: \"target/wit\",\n"); lib_rs.push_str(&format!(" world: \"{}\",\n", world_name)); @@ -645,7 +636,8 @@ crate-type = ["cdylib", "lib"] // Add global imports lib_rs.push_str("pub use hyperware_app_common::SendResult;\n"); lib_rs.push_str("pub use hyperware_app_common::send;\n"); - lib_rs.push_str("use hyperware_process_lib::Address;\n"); + lib_rs.push_str("use hyperware_app_common::hyperware_process_lib as hyperware_process_lib;\n"); + lib_rs.push_str("use hyperware_process_lib::{Address, Request};\n"); lib_rs.push_str("use serde_json::json;\n\n"); // Add interface use statements @@ -752,11 +744,11 @@ fn update_workspace_cargo_toml(base_dir: &Path) -> Result<()> { // Check if caller-utils is already in the members list let caller_utils_exists = members_array .iter() - .any(|m| m.as_str().map_or(false, |s| s == "caller-utils")); + .any(|m| m.as_str().map_or(false, |s| s == "crates/caller-utils")); if !caller_utils_exists { println!("Adding caller-utils to workspace members"); - members_array.push(Value::String("caller-utils".to_string())); + members_array.push(Value::String("crates/caller-utils".to_string())); // Write back the updated TOML let updated_content = toml::to_string_pretty(&parsed_toml) @@ -813,7 +805,7 @@ fn add_caller_utils_to_projects(projects: &[PathBuf]) -> Result<()> { let mut t = toml::map::Map::new(); t.insert( "path".to_string(), - Value::String("../caller-utils".to_string()), + Value::String("../crates/caller-utils".to_string()), ); t }), diff --git a/src/build/mod.rs b/src/build/mod.rs index 577c5892..8eb5044c 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -32,6 +32,7 @@ use crate::KIT_CACHE; mod rewrite; use rewrite::copy_and_rewrite_package; +mod caller_utils_generator; mod wit_generator; const PY_VENV_NAME: &str = "process_env"; @@ -1092,29 +1093,21 @@ async fn build_wit_dir( async fn compile_package_item( path: PathBuf, features: String, - apis: HashMap>, world: String, - wit_version: Option, + is_rust_process: bool, + is_py_process: bool, + is_js_process: bool, verbose: bool, ) -> Result<()> { - if path.is_dir() { - let is_rust_process = path.join(RUST_SRC_PATH).exists(); - let is_py_process = path.join(PYTHON_SRC_PATH).exists(); - let is_js_process = path.join(JAVASCRIPT_SRC_PATH).exists(); - if is_rust_process || is_py_process || is_js_process { - build_wit_dir(&path, &apis, wit_version).await?; - } - - if is_rust_process { - compile_rust_wasm_process(&path, &features, verbose).await?; - } else if is_py_process { - let python = get_python_version(None, None)? - .ok_or_else(|| eyre!("kit requires Python 3.10 or newer"))?; - compile_python_wasm_process(&path, &python, &world, verbose).await?; - } else if is_js_process { - let valid_node = get_newest_valid_node_version(None, None)?; - compile_javascript_wasm_process(&path, valid_node, &world, verbose).await?; - } + if is_rust_process { + compile_rust_wasm_process(&path, &features, verbose).await?; + } else if is_py_process { + let python = get_python_version(None, None)? + .ok_or_else(|| eyre!("kit requires Python 3.10 or newer"))?; + compile_python_wasm_process(&path, &python, &world, verbose).await?; + } else if is_js_process { + let valid_node = get_newest_valid_node_version(None, None)?; + compile_javascript_wasm_process(&path, valid_node, &world, verbose).await?; } Ok(()) } @@ -1539,6 +1532,7 @@ async fn compile_package( hyperapp: bool, force: bool, verbose: bool, + hyperapp_processed_projects: Option>, ignore_deps: bool, // for internal use; may cause problems when adding recursive deps ) -> Result<()> { let metadata = read_and_update_metadata(package_dir)?; @@ -1546,7 +1540,9 @@ async fn compile_package( let (mut apis, dependencies) = check_and_populate_dependencies(package_dir, &metadata, skip_deps_check, verbose).await?; + info!("dependencies: {dependencies:?}"); if !ignore_deps && !dependencies.is_empty() { + info!("fetching dependencies..."); fetch_dependencies( package_dir, &dependencies.iter().map(|s| s.to_string()).collect(), @@ -1564,7 +1560,7 @@ async fn compile_package( force, verbose, ) - .await?; + .await? } let wit_world = default_world @@ -1575,6 +1571,7 @@ async fn compile_package( let mut tasks = tokio::task::JoinSet::new(); let features = features.to_string(); + let mut to_compile = HashSet::new(); for entry in fs::read_dir(package_dir)? { let Ok(entry) = entry else { continue; @@ -1583,12 +1580,45 @@ async fn compile_package( if !is_cluded(&path, include, exclude) { continue; } + if !path.is_dir() { + continue; + } + + let is_rust_process = path.join(RUST_SRC_PATH).exists(); + let is_py_process = path.join(PYTHON_SRC_PATH).exists(); + let is_js_process = path.join(JAVASCRIPT_SRC_PATH).exists(); + if is_rust_process || is_py_process || is_js_process { + build_wit_dir(&path, &apis, metadata.properties.wit_version).await?; + } else { + continue; + } + to_compile.insert((path, is_rust_process, is_py_process, is_js_process)); + } + + // TODO: move process/target/wit -> target/wit + if !ignore_deps && !dependencies.is_empty() { + info!("{hyperapp_processed_projects:?}"); + if let Some(ref processed_projects) = hyperapp_processed_projects { + for processed_project in processed_projects { + let api_dir = processed_project.join("target").join("wit"); + info!("{processed_project:?} {api_dir:?}"); + caller_utils_generator::create_caller_utils( + package_dir, + &api_dir, + &[processed_project.clone()], + )?; + } + } + } + + for (path, is_rust_process, is_py_process, is_js_process) in to_compile { tasks.spawn(compile_package_item( path, features.clone(), - apis.clone(), wit_world.clone(), - metadata.properties.wit_version, + is_rust_process, + is_py_process, + is_js_process, verbose.clone(), )); } @@ -1761,11 +1791,18 @@ pub async fn execute( copy_and_rewrite_package(package_dir)? }; - if hyperapp { + let hyperapp_processed_projects = if !hyperapp { + None + } else { let api_dir = live_dir.join("api"); - let (_processed_projects, _interfaces) = + let (processed_projects, interfaces) = wit_generator::generate_wit_files(&live_dir, &api_dir, false)?; - } + if interfaces.is_empty() { + None + } else { + Some(processed_projects) + } + }; let ui_dirs = get_ui_dirs(&live_dir, &include, &exclude)?; if !no_ui && !ui_dirs.is_empty() { @@ -1796,6 +1833,7 @@ pub async fn execute( hyperapp, force, verbose, + hyperapp_processed_projects, ignore_deps, ) .await?; diff --git a/src/build/wit_generator.rs b/src/build/wit_generator.rs index 4971792a..76a446bb 100644 --- a/src/build/wit_generator.rs +++ b/src/build/wit_generator.rs @@ -155,6 +155,29 @@ fn rust_type_to_wit(ty: &Type, used_types: &mut HashSet) -> Result { + if let syn::PathArguments::AngleBracketed(args) = + &type_path.path.segments.last().unwrap().arguments + { + if args.args.len() >= 2 { + if let ( + Some(syn::GenericArgument::Type(ok_ty)), + Some(syn::GenericArgument::Type(err_ty)), + ) = (args.args.first(), args.args.get(1)) + { + let ok_type = rust_type_to_wit(ok_ty, used_types)?; + let err_type = rust_type_to_wit(err_ty, used_types)?; + Ok(format!("result<{}, {}>", ok_type, err_type)) + } else { + Err(eyre!("Failed to parse Result generic arguments")) + } + } else { + Err(eyre!("Result requires two type arguments")) + } + } else { + Err(eyre!("Failed to parse Result type arguments")) + } + } // TODO: fix and enable //"HashMap" | "BTreeMap" => { // if let syn::PathArguments::AngleBracketed(args) = @@ -920,7 +943,7 @@ fn rewrite_wit( println!("Extracted world name: {}", world_name); // Check if this world name matches the one we're looking for - if wit_worlds.remove(&world_name) { + if wit_worlds.remove(&world_name) || wit_worlds.contains(&world_name[6..]) { // Determine the include line based on world name // If world name starts with "types-", use "include lib;" instead if world_name.starts_with("types-") { @@ -1052,6 +1075,20 @@ pub fn generate_wit_files( ) })?; + let new_file_path = api_dir.join(format!("types-{}.wit", wit_world)); + let simple_world_content = format!("world types-{} {{}}", wit_world); + + println!( + "Creating new world definition file: {}", + new_file_path.display() + ); + fs::write(&new_file_path, simple_world_content).with_context(|| { + format!( + "Failed to create new world file: {}", + new_file_path.display() + ) + })?; + println!("Successfully created new world definition file"); updated_world = true; } From 39842593e3cd4288823da93bff3658d7e1fdd84a Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 8 Apr 2025 21:49:39 -0700 Subject: [PATCH 8/8] build: use latest macro --- src/build/caller_utils_generator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/caller_utils_generator.rs b/src/build/caller_utils_generator.rs index aaeed467..84d6109b 100644 --- a/src/build/caller_utils_generator.rs +++ b/src/build/caller_utils_generator.rs @@ -495,7 +495,7 @@ process_macros = "0.1.0" futures-util = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -hyperware_app_common = { git = "https://github.com/hyperware-ai/hyperprocess-macro", rev = "4e417e1" } +hyperware_app_common = { git = "https://github.com/hyperware-ai/hyperprocess-macro", rev = "f29952f" } once_cell = "1.20.2" futures = "0.3" uuid = { version = "1.0" }