From 51554458ecccf81d5fdcc9a3d52117b1aac128a0 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 13:44:02 +0530 Subject: [PATCH 01/55] embed_internal_str with logs without making any changes --- macros/src/lib.rs | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index f50ac8c..4a775f9 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -687,7 +687,7 @@ impl<'ast> SupportedVisitItem<'ast> for ItemVisitor { } } -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] enum ResultStyle { Export, ExportContent, @@ -942,34 +942,59 @@ fn source_excerpt<'a, T: ToTokens>( /// Inner version of [`embed_internal`] that just returns the result as a [`String`]. fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) -> Result { - let args = parse2::(tokens.into())?; + let args: EmbedArgs = parse2::(tokens.into())?; + println!( + "embed_internal_str ----> args: file_path: {}, item_ident: {:?}", + args.file_path.value(), + args.item_ident.as_ref().map(|i| i.to_string()) + ); // return blank result if we can't properly resolve `caller_crate_root` let Some(root) = caller_crate_root() else { + println!("embed_internal_str ----> Failed to resolve caller_crate_root"); return Ok(String::from("")); }; + println!("embed_internal_str ----> Root resolved: {:?}", root); let file_path = root.join(args.file_path.value()); + println!("embed_internal_str ----> File path: {:?}", file_path); let source_code = match fs::read_to_string(&file_path) { - Ok(src) => src, - Err(_) => { + Ok(src) => { + println!("embed_internal_str ----> Successfully read source file"); + src + } + Err(e) => { + println!( + "embed_internal_str ----> Failed to read source file: {:?}", + e + ); return Err(Error::new( args.file_path.span(), format!( "Could not read the specified path '{}'.", file_path.display(), ), - )) + )); } }; let parsed = source_code.parse::()?; let source_file = parse2::(parsed)?; + println!("embed_internal_str ----> Parsed source file successfully"); let output = if let Some(ident) = args.item_ident { + println!("embed_internal_str ----> Searching for item: {}", ident); let mut visitor = ItemVisitor { search: ident.clone(), results: Vec::new(), }; visitor.visit_file(&source_file); + println!( + "embed_internal_str ----> Visitor results: {:?}", + visitor.results + ); if visitor.results.is_empty() { + println!( + "embed_internal_str ----> No results found for item: {}", + ident + ); return Err(Error::new( ident.span(), format!( @@ -981,6 +1006,10 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } let mut results: Vec = Vec::new(); for (item, style) in visitor.results { + println!( + "embed_internal_str ----> Processing item with style: {:?}", + style + ); let excerpt = source_excerpt(&source_code, &item, style)?; let formatted = fix_indentation(excerpt); let example = into_example(formatted.as_str(), lang); @@ -988,8 +1017,13 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } results.join("\n") } else { + println!("embed_internal_str ----> No specific item requested, using entire source"); into_example(source_code.as_str(), lang) }; + println!( + "embed_internal_str ----> Final output length: {}", + output.len() + ); Ok(output) } From e733fd7fcc4150c1803e9529f95d954b7262d66a Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 14:32:27 +0530 Subject: [PATCH 02/55] added more debug statements in embed internal string --- macros/src/lib.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 4a775f9..63c7f3b 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -959,6 +959,10 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let source_code = match fs::read_to_string(&file_path) { Ok(src) => { println!("embed_internal_str ----> Successfully read source file"); + println!( + "embed_internal_str ----> Source code of the entire file: {}", + src + ); src } Err(e) => { @@ -976,8 +980,27 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } }; let parsed = source_code.parse::()?; - let source_file = parse2::(parsed)?; + println!("embed_internal_str ----> Parsed source code successfully"); + println!( + "embed_internal_str ----> Parsed source code: removed comments and other cleanups {}", + parsed + ); + let source_file: File = parse2::(parsed)?; println!("embed_internal_str ----> Parsed source file successfully"); + println!( + "embed_internal_str ----> Source file items count: {}", + source_file.items.len() + ); + println!( + "embed_internal_str ----> Source file attributes count: {}", + source_file.attrs.len() + ); + if let Some(shebang) = &source_file.shebang { + println!( + "embed_internal_str ----> Source file has shebang: {}", + shebang + ); + } let output = if let Some(ident) = args.item_ident { println!("embed_internal_str ----> Searching for item: {}", ident); @@ -1011,8 +1034,11 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - style ); let excerpt = source_excerpt(&source_code, &item, style)?; + println!("embed_internal_str ----> Excerpt: {}", excerpt); let formatted = fix_indentation(excerpt); + println!("embed_internal_str ----> Formatted: {}", formatted); let example = into_example(formatted.as_str(), lang); + println!("embed_internal_str ----> Example: {}", example); results.push(example); } results.join("\n") From d7156d3a99636941b21b6024aa8d7d252ced5ebb Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 15:10:40 +0530 Subject: [PATCH 03/55] added named embedargs --- macros/src/lib.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 63c7f3b..83784bd 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -15,6 +15,9 @@ use std::{ path::{Path, PathBuf}, str::FromStr, }; +use syn::custom_keyword; +use syn::parse::Parse; +use syn::parse::ParseStream; use syn::{ parse2, spanned::Spanned, @@ -541,14 +544,85 @@ pub fn embed_run(tokens: TokenStream) -> TokenStream { } /// Used to parse args for `docify::embed!(..)` -#[derive(Parse)] +/// Used to parse args for `docify::embed!(..)` +/// Used to parse args for `docify::embed!(..)` + +mod kw { + syn::custom_keyword!(git); + syn::custom_keyword!(path); + syn::custom_keyword!(item); +} + struct EmbedArgs { + git_url: Option, file_path: LitStr, - #[prefix(Option as comma)] - #[parse_if(comma.is_some())] item_ident: Option, } +impl Parse for EmbedArgs { + fn parse(input: ParseStream) -> Result { + println!("impl Parse for EmbedArgs: ----> Starting parse function"); + let mut git_url = None; + let mut file_path = None; + let mut item_ident = None; + + while !input.is_empty() { + println!("impl Parse for EmbedArgs: ----> Entering loop iteration"); + let lookahead = input.lookahead1(); + if lookahead.peek(kw::git) { + println!("impl Parse for EmbedArgs: ----> Parsing git URL"); + input.parse::()?; + input.parse::()?; + git_url = Some(input.parse::()?); + println!( + "impl Parse for EmbedArgs: ----> Git URL parsed: {}", + git_url.as_ref().map(LitStr::value).unwrap_or_default() + ); + } else if lookahead.peek(kw::path) { + println!("impl Parse for EmbedArgs: ----> Parsing file path"); + input.parse::()?; + input.parse::()?; + file_path = Some(input.parse::()?); + println!( + "impl Parse for EmbedArgs: ----> File path parsed: {}", + file_path.as_ref().map(LitStr::value).unwrap_or_default() + ); + } else if lookahead.peek(kw::item) { + println!("impl Parse for EmbedArgs: ----> Parsing item identifier"); + input.parse::()?; + input.parse::()?; + item_ident = Some(input.parse::()?); + println!( + "impl Parse for EmbedArgs: ----> Item identifier parsed: {:?}", + item_ident + ); + } else { + println!("impl Parse for EmbedArgs: ----> Encountered unexpected token"); + return Err(lookahead.error()); + } + + if !input.is_empty() { + println!("impl Parse for EmbedArgs: ----> Parsing comma separator"); + input.parse::()?; + } + } + + println!("impl Parse for EmbedArgs: ----> Finished parsing all arguments"); + let file_path = file_path.ok_or_else(|| input.error("expected `path` argument"))?; + println!( + "impl Parse for EmbedArgs: ----> Final file path: {}", + file_path.value() + ); + + println!("impl Parse for EmbedArgs: ----> Returning parsed EmbedArgs"); + Ok(EmbedArgs { + git_url, + file_path, + item_ident, + }) + } +} + impl ToTokens for EmbedArgs { fn to_tokens(&self, tokens: &mut TokenStream2) { tokens.extend(self.file_path.to_token_stream()); @@ -944,7 +1018,8 @@ fn source_excerpt<'a, T: ToTokens>( fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) -> Result { let args: EmbedArgs = parse2::(tokens.into())?; println!( - "embed_internal_str ----> args: file_path: {}, item_ident: {:?}", + "embed_internal_str ----> args: git_url: {:?}, file_path: {}, item_ident: {:?}", + args.git_url.as_ref().map(|url| url.value()), args.file_path.value(), args.item_ident.as_ref().map(|i| i.to_string()) ); From 7472229baa970709adb0a362e02234841b666cb7 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 15:18:59 +0530 Subject: [PATCH 04/55] forbid use of "../" in file_path --- macros/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 83784bd..f438e1c 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1023,6 +1023,14 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - args.file_path.value(), args.item_ident.as_ref().map(|i| i.to_string()) ); + + // Check if the file_path starts with "../" and git_url is not provided + if args.file_path.value().starts_with("../") && args.git_url.is_none() { + return Err(Error::new( + args.file_path.span(), + "You can only embed files which are present in the current crate. For any other files, please provide the git_url to embed." + )); + } // return blank result if we can't properly resolve `caller_crate_root` let Some(root) = caller_crate_root() else { println!("embed_internal_str ----> Failed to resolve caller_crate_root"); From cc1d56f6abf32ec87eeee45a486fac5f183eaf61 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 15:29:13 +0530 Subject: [PATCH 05/55] added check where git url is passed but path starts with ".." or "/" in that case throw an error that please pass the correct source file path --- macros/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index f438e1c..8c15bd6 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1031,6 +1031,17 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - "You can only embed files which are present in the current crate. For any other files, please provide the git_url to embed." )); } + + // New check: If git_url is provided, ensure the path doesn't start with ".." or "../" + if args.git_url.is_some() + && (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("../")) + { + return Err(Error::new( + args.file_path.span(), + "When using git_url, please provide the correct file path in your git source. The path should not start with '..' or '../'." + )); + } + // return blank result if we can't properly resolve `caller_crate_root` let Some(root) = caller_crate_root() else { println!("embed_internal_str ----> Failed to resolve caller_crate_root"); From a3b5e966ab58050f0ac6804f55b192c0e21e3aff Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 16:37:53 +0530 Subject: [PATCH 06/55] clone working --- macros/Cargo.toml | 2 ++ macros/src/lib.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 173e9ab..0520c5e 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -22,3 +22,5 @@ common-path = "1" termcolor = "1" once_cell = "1" toml = "0.8" +tempfile = "3.3.0" +git2 = "0.16.0" \ No newline at end of file diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 8c15bd6..be6411d 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,6 +2,7 @@ use common_path::common_path; use derive_syn_parse::Parse; +use git2::{FetchOptions, RemoteCallbacks, Repository}; use once_cell::sync::Lazy; use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; @@ -26,6 +27,7 @@ use syn::{ AttrStyle, Attribute, Error, File, Ident, ImplItem, Item, LitStr, Meta, Result, Token, TraitItem, }; +use tempfile::TempDir; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use toml::{Table, Value}; use walkdir::WalkDir; @@ -1047,8 +1049,18 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - println!("embed_internal_str ----> Failed to resolve caller_crate_root"); return Ok(String::from("")); }; + + let file_path = if let Some(git_url) = &args.git_url { + let repo_path = clone_repo(git_url.value().as_str(), &root)?; + repo_path.join(args.file_path.value()) + } else { + root.join(args.file_path.value()) + }; + + println!("embed_internal_str ----> File path: {:?}", file_path); + println!("embed_internal_str ----> Root resolved: {:?}", root); - let file_path = root.join(args.file_path.value()); + // let file_path = root.join(args.file_path.value()); println!("embed_internal_str ----> File path: {:?}", file_path); let source_code = match fs::read_to_string(&file_path) { Ok(src) => { @@ -1338,6 +1350,51 @@ fn compile_markdown_dir, P2: AsRef>( Ok(()) } +fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { + // Create a temporary directory within the project root + let temp_dir = TempDir::new_in(project_root).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create temporary directory: {}", e), + ) + })?; + + println!( + "Temporary directory created at: {}", + temp_dir.path().display() + ); + + // Set up fetch options + // let mut fetch_opts = FetchOptions::new(); + // fetch_opts.depth(1); // Shallow clone (only latest commit) + + // Set up remote callbacks + let mut callbacks = RemoteCallbacks::new(); + callbacks.transfer_progress(|progress| { + println!( + "Transfer progress: {}/{} objects", + progress.received_objects(), + progress.total_objects() + ); + true + }); + // fetch_opts.remote_callbacks(callbacks); + + // Clone the repository + let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to clone repository: {}", e), + ) + })?; + + println!("Repository cloned successfully"); + + println!("Repository cloned to: {}", temp_dir.path().display()); + + Ok(temp_dir.into_path()) +} + /// Docifies the specified markdown source string fn compile_markdown_source>(source: S) -> Result { let source = source.as_ref(); From 48e4f81d4c7100875bb0ed5df51c1d8f22bbaf04 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 18:19:27 +0530 Subject: [PATCH 07/55] hash part working , branch , tag part remaining , network check part remaining , edge cases remaining --- macros/Cargo.toml | 3 +- macros/src/lib.rs | 314 ++++++++++++++++++++++++++++++---------------- 2 files changed, 208 insertions(+), 109 deletions(-) diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 0520c5e..eb61498 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -23,4 +23,5 @@ termcolor = "1" once_cell = "1" toml = "0.8" tempfile = "3.3.0" -git2 = "0.16.0" \ No newline at end of file +git2 = "0.16.0" +sha2 = "0.10.6" diff --git a/macros/src/lib.rs b/macros/src/lib.rs index be6411d..27785a0 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,12 +2,13 @@ use common_path::common_path; use derive_syn_parse::Parse; -use git2::{FetchOptions, RemoteCallbacks, Repository}; +use git2::{RemoteCallbacks, Repository}; use once_cell::sync::Lazy; use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, ToTokens}; use regex::Regex; +use sha2::{Digest, Sha256}; use std::{ cmp::min, collections::HashMap, @@ -16,7 +17,6 @@ use std::{ path::{Path, PathBuf}, str::FromStr, }; -use syn::custom_keyword; use syn::parse::Parse; use syn::parse::ParseStream; use syn::{ @@ -1016,7 +1016,6 @@ fn source_excerpt<'a, T: ToTokens>( .join("\n")) } -/// Inner version of [`embed_internal`] that just returns the result as a [`String`]. fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) -> Result { let args: EmbedArgs = parse2::(tokens.into())?; println!( @@ -1034,9 +1033,9 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - )); } - // New check: If git_url is provided, ensure the path doesn't start with ".." or "../" + // Check if git_url is provided and path starts with ".." or "../" if args.git_url.is_some() - && (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("../")) + && (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("/")) { return Err(Error::new( args.file_path.span(), @@ -1044,121 +1043,100 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - )); } - // return blank result if we can't properly resolve `caller_crate_root` - let Some(root) = caller_crate_root() else { - println!("embed_internal_str ----> Failed to resolve caller_crate_root"); - return Ok(String::from("")); - }; + let crate_root = caller_crate_root() + .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; - let file_path = if let Some(git_url) = &args.git_url { - let repo_path = clone_repo(git_url.value().as_str(), &root)?; - repo_path.join(args.file_path.value()) - } else { - root.join(args.file_path.value()) - }; + if let Some(git_url) = &args.git_url { + println!("Detected git-based embedding"); + let repo_path = clone_repo(git_url.value().as_str(), &crate_root)?; + let file_path = args.file_path.value().to_string(); - println!("embed_internal_str ----> File path: {:?}", file_path); + let full_path = repo_path.join(&file_path).to_string_lossy().into_owned(); + println!("embed_internal_str ----> Full path: {}", full_path); - println!("embed_internal_str ----> Root resolved: {:?}", root); - // let file_path = root.join(args.file_path.value()); - println!("embed_internal_str ----> File path: {:?}", file_path); - let source_code = match fs::read_to_string(&file_path) { - Ok(src) => { - println!("embed_internal_str ----> Successfully read source file"); - println!( - "embed_internal_str ----> Source code of the entire file: {}", - src - ); - src - } - Err(e) => { - println!( - "embed_internal_str ----> Failed to read source file: {:?}", - e - ); - return Err(Error::new( + let snippet_content = match &args.item_ident { + Some(ident) => { + println!("Generating snippet for item: {}", ident); + manage_snippet(&crate_root, &full_path, &ident.to_string())? + } + None => { + println!("No specific item requested, using entire file content"); + fs::read_to_string(repo_path.join(&file_path)).map_err(|e| { + Error::new(Span::call_site(), format!("Failed to read file: {}", e)) + })? + } + }; + + let formatted = fix_indentation(&snippet_content); + let output = into_example(&formatted, lang); + + println!("Successfully embedded git-based content"); + println!( + "embed_internal_str ----> Final output length: {}", + output.len() + ); + + Ok(output) + } else { + println!("Detected local file embedding"); + let file_path = crate_root.join(args.file_path.value()); + println!("embed_internal_str ----> File path: {:?}", file_path); + + let source_code = fs::read_to_string(&file_path).map_err(|e| { + Error::new( args.file_path.span(), format!( - "Could not read the specified path '{}'.", + "Could not read the specified path '{}': {}", file_path.display(), + e ), - )); - } - }; - let parsed = source_code.parse::()?; - println!("embed_internal_str ----> Parsed source code successfully"); - println!( - "embed_internal_str ----> Parsed source code: removed comments and other cleanups {}", - parsed - ); - let source_file: File = parse2::(parsed)?; - println!("embed_internal_str ----> Parsed source file successfully"); - println!( - "embed_internal_str ----> Source file items count: {}", - source_file.items.len() - ); - println!( - "embed_internal_str ----> Source file attributes count: {}", - source_file.attrs.len() - ); - if let Some(shebang) = &source_file.shebang { - println!( - "embed_internal_str ----> Source file has shebang: {}", - shebang - ); - } + ) + })?; - let output = if let Some(ident) = args.item_ident { - println!("embed_internal_str ----> Searching for item: {}", ident); - let mut visitor = ItemVisitor { - search: ident.clone(), - results: Vec::new(), + let source_file = syn::parse_file(&source_code)?; + println!("embed_internal_str ----> Parsed source file successfully"); + + let output = if let Some(ident) = args.item_ident.as_ref() { + println!("embed_internal_str ----> Searching for item: {}", ident); + let mut visitor = ItemVisitor { + search: ident.clone(), + results: Vec::new(), + }; + visitor.visit_file(&source_file); + + if visitor.results.is_empty() { + return Err(Error::new( + ident.span(), + format!( + "Could not find docify export item '{}' in '{}'.", + ident, + file_path.display() + ), + )); + } + + let mut results: Vec = Vec::new(); + for (item, style) in visitor.results { + let excerpt = source_excerpt(&source_code, &item, style)?; + let formatted = fix_indentation(excerpt); + let example = into_example(formatted.as_str(), lang); + results.push(example); + } + results.join("\n") + } else { + println!("embed_internal_str ----> No specific item requested, using entire source"); + into_example(source_code.as_str(), lang) }; - visitor.visit_file(&source_file); + + println!("Successfully embedded local content"); println!( - "embed_internal_str ----> Visitor results: {:?}", - visitor.results + "embed_internal_str ----> Final output length: {}", + output.len() ); - if visitor.results.is_empty() { - println!( - "embed_internal_str ----> No results found for item: {}", - ident - ); - return Err(Error::new( - ident.span(), - format!( - "Could not find docify export item '{}' in '{}'.", - ident, - file_path.display(), - ), - )); - } - let mut results: Vec = Vec::new(); - for (item, style) in visitor.results { - println!( - "embed_internal_str ----> Processing item with style: {:?}", - style - ); - let excerpt = source_excerpt(&source_code, &item, style)?; - println!("embed_internal_str ----> Excerpt: {}", excerpt); - let formatted = fix_indentation(excerpt); - println!("embed_internal_str ----> Formatted: {}", formatted); - let example = into_example(formatted.as_str(), lang); - println!("embed_internal_str ----> Example: {}", example); - results.push(example); - } - results.join("\n") - } else { - println!("embed_internal_str ----> No specific item requested, using entire source"); - into_example(source_code.as_str(), lang) - }; - println!( - "embed_internal_str ----> Final output length: {}", - output.len() - ); - Ok(output) -} + Ok(output) + } +} /// Internal implementation behind [`macro@embed`]. fn embed_internal(tokens: impl Into, lang: MarkdownLanguage) -> Result { let output = embed_internal_str(tokens, lang)?; @@ -1394,6 +1372,126 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { Ok(temp_dir.into_path()) } +fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { + let full_path = crate_root.join(file_path); + println!( + "inside manage_snippet ----> Full path to crate root: {}", + crate_root.display() + ); + println!( + "inside manage_snippet ----> Full path to file: {}", + full_path.display() + ); + + let snippets_dir = crate_root.join(".snippets"); + fs::create_dir_all(&snippets_dir).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create .snippets directory: {}", e), + ) + })?; + + let snippet_path = generate_snippet_path(&snippets_dir, file_path, item_ident); + println!("Snippet path: {}", snippet_path.display()); + + let is_docs_rs = std::env::var("DOCS_RS").is_ok(); + + if is_docs_rs { + println!("Running on docs.rs, reading existing snippet"); + fs::read_to_string(&snippet_path).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to read snippet file: {}", e), + ) + }) + } else { + println!("Local development, checking if snippet needs updating"); + let existing_content = fs::read_to_string(&snippet_path).ok(); + let new_content = extract_item_from_file(&full_path, item_ident)?; + + if existing_content.as_ref().map(|c| hash_content(c)) != Some(hash_content(&new_content)) { + println!("Updating snippet file"); + fs::write(&snippet_path, &new_content).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to write snippet file: {}", e), + ) + })?; + } else { + println!("Snippet is up to date"); + } + + Ok(new_content) + } +} + +fn generate_snippet_path(snippets_dir: &Path, file_path: &str, item_ident: &str) -> PathBuf { + println!( + "inside generate_snippet_path ----> Snippets directory: {}", + snippets_dir.display() + ); + println!( + "inside generate_snippet_path ----> File path: {}", + file_path + ); + println!( + "inside generate_snippet_path ----> Item ident: {}", + item_ident + ); + let path = PathBuf::from(file_path); + let file_name = path + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or("unknown"); + snippets_dir.join(format!(".docify-snippet-{}-{}", file_name, item_ident)) +} +fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { + println!( + "Extracting item '{}' from '{}'", + item_ident, + file_path.display() + ); + + let source_code = fs::read_to_string(file_path).map_err(|e| { + Error::new( + Span::call_site(), + format!( + "Could not read the specified path '{}': {}", + file_path.display(), + e + ), + ) + })?; + + let mut visitor = ItemVisitor { + search: syn::parse_str(item_ident)?, + results: Vec::new(), + }; + visitor.visit_file(&syn::parse_file(&source_code)?); + + if visitor.results.is_empty() { + return Err(Error::new( + Span::call_site(), + format!( + "Could not find docify export item '{}' in '{}'.", + item_ident, + file_path.display() + ), + )); + } + + println!("Successfully extracted item from file"); + let (item, style) = visitor.results.first().unwrap(); + source_excerpt(&source_code, item, *style) +} + +fn hash_content(content: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(content); + let result = format!("{:x}", hasher.finalize()); + println!("Content hash: {}", result); + result +} /// Docifies the specified markdown source string fn compile_markdown_source>(source: S) -> Result { From 663b376bcf5262ea5c0d504350e4ae913122c1b8 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 19:05:24 +0530 Subject: [PATCH 08/55] added more logs --- macros/src/lib.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 27785a0..d7a98f2 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1372,6 +1372,7 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { Ok(temp_dir.into_path()) } + fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { let full_path = crate_root.join(file_path); println!( @@ -1410,7 +1411,7 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul let new_content = extract_item_from_file(&full_path, item_ident)?; if existing_content.as_ref().map(|c| hash_content(c)) != Some(hash_content(&new_content)) { - println!("Updating snippet file"); + println!("inside manage_snippet ----> Updating snippet file"); fs::write(&snippet_path, &new_content).map_err(|e| { Error::new( Span::call_site(), @@ -1418,7 +1419,7 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul ) })?; } else { - println!("Snippet is up to date"); + println!("inside manage_snippet ----> Snippet is up to date"); } Ok(new_content) @@ -1447,7 +1448,7 @@ fn generate_snippet_path(snippets_dir: &Path, file_path: &str, item_ident: &str) } fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { println!( - "Extracting item '{}' from '{}'", + "inside extract_item_from_file ----> Extracting item '{}' from '{}'", item_ident, file_path.display() ); @@ -1463,12 +1464,20 @@ fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result ) })?; + println!( + "inside extract_item_from_file ----> Source code: {}", + source_code + ); + let mut visitor = ItemVisitor { search: syn::parse_str(item_ident)?, results: Vec::new(), }; visitor.visit_file(&syn::parse_file(&source_code)?); - + println!( + "inside extract_item_from_file ----> Visitor results: {:?}", + visitor.results + ); if visitor.results.is_empty() { return Err(Error::new( Span::call_site(), From 6430206abe815b64e6cec4f4109f9fd8bd72a65c Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 20:32:31 +0530 Subject: [PATCH 09/55] clone all the remote branch and switch to a particular branch working , currently switching to docif --- macros/src/lib.rs | 59 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index d7a98f2..af2efd5 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,7 +2,7 @@ use common_path::common_path; use derive_syn_parse::Parse; -use git2::{RemoteCallbacks, Repository}; +use git2::{FetchOptions, RemoteCallbacks, Repository}; use once_cell::sync::Lazy; use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; @@ -1342,10 +1342,6 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { temp_dir.path().display() ); - // Set up fetch options - // let mut fetch_opts = FetchOptions::new(); - // fetch_opts.depth(1); // Shallow clone (only latest commit) - // Set up remote callbacks let mut callbacks = RemoteCallbacks::new(); callbacks.transfer_progress(|progress| { @@ -1356,9 +1352,12 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { ); true }); - // fetch_opts.remote_callbacks(callbacks); - // Clone the repository + // Set up fetch options + let mut fetch_opts = FetchOptions::new(); + fetch_opts.remote_callbacks(callbacks); + + // Clone the repository with all branches let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { Error::new( Span::call_site(), @@ -1367,9 +1366,53 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { })?; println!("Repository cloned successfully"); - println!("Repository cloned to: {}", temp_dir.path().display()); + // Fetch all branches + let mut remote = repo.find_remote("origin").map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find remote 'origin': {}", e), + ) + })?; + + remote + .fetch(&["refs/heads/*:refs/heads/*"], Some(&mut fetch_opts), None) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to fetch all branches: {}", e), + ) + })?; + + // Switch to the "docif" branch + let branch_name = "docif"; + let (object, reference) = repo + .revparse_ext(&format!("origin/{}", branch_name)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find '{}' branch: {}", branch_name, e), + ) + })?; + + repo.checkout_tree(&object, None).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to checkout '{}' branch: {}", branch_name, e), + ) + })?; + + repo.set_head(&format!("refs/heads/{}", branch_name)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to '{}' branch: {}", branch_name, e), + ) + })?; + + println!("Switched to '{}' branch", branch_name); + Ok(temp_dir.into_path()) } From 36aea01bfa2071f340885a8ec609016a6175d28d Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 20:38:43 +0530 Subject: [PATCH 10/55] added branch_name optional param to clone_repo , if passed None it only clones the default repo --- macros/src/lib.rs | 81 ++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index af2efd5..8512192 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1048,7 +1048,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(git_url) = &args.git_url { println!("Detected git-based embedding"); - let repo_path = clone_repo(git_url.value().as_str(), &crate_root)?; + let repo_path = clone_repo(git_url.value().as_str(), &crate_root, None)?; let file_path = args.file_path.value().to_string(); let full_path = repo_path.join(&file_path).to_string_lossy().into_owned(); @@ -1328,7 +1328,7 @@ fn compile_markdown_dir, P2: AsRef>( Ok(()) } -fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { +fn clone_repo(git_url: &str, project_root: &PathBuf, branch_name: Option<&str>) -> Result { // Create a temporary directory within the project root let temp_dir = TempDir::new_in(project_root).map_err(|e| { Error::new( @@ -1353,11 +1353,7 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { true }); - // Set up fetch options - let mut fetch_opts = FetchOptions::new(); - fetch_opts.remote_callbacks(callbacks); - - // Clone the repository with all branches + // Clone the repository let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { Error::new( Span::call_site(), @@ -1368,50 +1364,57 @@ fn clone_repo(git_url: &str, project_root: &PathBuf) -> Result { println!("Repository cloned successfully"); println!("Repository cloned to: {}", temp_dir.path().display()); - // Fetch all branches - let mut remote = repo.find_remote("origin").map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find remote 'origin': {}", e), - ) - })?; + if let Some(branch) = branch_name { + // Set up fetch options + let mut fetch_opts = FetchOptions::new(); + fetch_opts.remote_callbacks(callbacks); - remote - .fetch(&["refs/heads/*:refs/heads/*"], Some(&mut fetch_opts), None) - .map_err(|e| { + // Fetch all branches + let mut remote = repo.find_remote("origin").map_err(|e| { Error::new( Span::call_site(), - format!("Failed to fetch all branches: {}", e), + format!("Failed to find remote 'origin': {}", e), ) })?; - // Switch to the "docif" branch - let branch_name = "docif"; - let (object, reference) = repo - .revparse_ext(&format!("origin/{}", branch_name)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find '{}' branch: {}", branch_name, e), - ) - })?; - - repo.checkout_tree(&object, None).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout '{}' branch: {}", branch_name, e), - ) - })?; + remote + .fetch(&["refs/heads/*:refs/heads/*"], Some(&mut fetch_opts), None) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to fetch all branches: {}", e), + ) + })?; - repo.set_head(&format!("refs/heads/{}", branch_name)) - .map_err(|e| { + // Switch to the specified branch + let (object, reference) = + repo.revparse_ext(&format!("origin/{}", branch)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find '{}' branch: {}", branch, e), + ) + })?; + + repo.checkout_tree(&object, None).map_err(|e| { Error::new( Span::call_site(), - format!("Failed to set HEAD to '{}' branch: {}", branch_name, e), + format!("Failed to checkout '{}' branch: {}", branch, e), ) })?; - println!("Switched to '{}' branch", branch_name); + repo.set_head(&format!("refs/heads/{}", branch)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to '{}' branch: {}", branch, e), + ) + })?; + + println!("Switched to '{}' branch", branch); + } else { + println!("Cloned default branch only"); + } Ok(temp_dir.into_path()) } From 8a7e8dd904cab4dc8c658b7cc4c12b097049be21 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 21:01:03 +0530 Subject: [PATCH 11/55] added commit hash when passed it check's out that specific commit hash , still need to handle the case where both branch and commit hash is provided --- macros/src/lib.rs | 82 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 8512192..eaf04c8 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,7 +2,7 @@ use common_path::common_path; use derive_syn_parse::Parse; -use git2::{FetchOptions, RemoteCallbacks, Repository}; +use git2::{FetchOptions, Oid, RemoteCallbacks, Repository}; use once_cell::sync::Lazy; use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; @@ -1048,7 +1048,12 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(git_url) = &args.git_url { println!("Detected git-based embedding"); - let repo_path = clone_repo(git_url.value().as_str(), &crate_root, None)?; + let repo_path = clone_repo( + git_url.value().as_str(), + &crate_root, + None, + Some("4a9108d93525db4be232c5d03998fd7dadcd65c9"), + )?; let file_path = args.file_path.value().to_string(); let full_path = repo_path.join(&file_path).to_string_lossy().into_owned(); @@ -1328,7 +1333,12 @@ fn compile_markdown_dir, P2: AsRef>( Ok(()) } -fn clone_repo(git_url: &str, project_root: &PathBuf, branch_name: Option<&str>) -> Result { +fn clone_repo( + git_url: &str, + project_root: &PathBuf, + branch_name: Option<&str>, + commit_hash: Option<&str>, +) -> Result { // Create a temporary directory within the project root let temp_dir = TempDir::new_in(project_root).map_err(|e| { Error::new( @@ -1364,7 +1374,7 @@ fn clone_repo(git_url: &str, project_root: &PathBuf, branch_name: Option<&str>) println!("Repository cloned successfully"); println!("Repository cloned to: {}", temp_dir.path().display()); - if let Some(branch) = branch_name { + if branch_name.is_some() || commit_hash.is_some() { // Set up fetch options let mut fetch_opts = FetchOptions::new(); fetch_opts.remote_callbacks(callbacks); @@ -1386,32 +1396,66 @@ fn clone_repo(git_url: &str, project_root: &PathBuf, branch_name: Option<&str>) ) })?; - // Switch to the specified branch - let (object, reference) = - repo.revparse_ext(&format!("origin/{}", branch)) + if let Some(commit) = commit_hash { + // Checkout the specific commit + let oid = Oid::from_str(commit).map_err(|e| { + Error::new( + Span::call_site(), + format!("Invalid commit hash '{}': {}", commit, e), + ) + })?; + + let commit_obj = repo.find_commit(oid).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find commit '{}': {}", commit, e), + ) + })?; + + repo.set_head_detached(commit_obj.id()).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to commit '{}': {}", commit, e), + ) + })?; + + repo.checkout_head(Some(git2::build::CheckoutBuilder::new().force())) .map_err(|e| { Error::new( Span::call_site(), - format!("Failed to find '{}' branch: {}", branch, e), + format!("Failed to checkout commit '{}': {}", commit, e), ) })?; - repo.checkout_tree(&object, None).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout '{}' branch: {}", branch, e), - ) - })?; - - repo.set_head(&format!("refs/heads/{}", branch)) - .map_err(|e| { + println!("Checked out commit '{}'", commit); + } else if let Some(branch) = branch_name { + // Switch to the specified branch + let (object, reference) = + repo.revparse_ext(&format!("origin/{}", branch)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find '{}' branch: {}", branch, e), + ) + })?; + + repo.checkout_tree(&object, None).map_err(|e| { Error::new( Span::call_site(), - format!("Failed to set HEAD to '{}' branch: {}", branch, e), + format!("Failed to checkout '{}' branch: {}", branch, e), ) })?; - println!("Switched to '{}' branch", branch); + repo.set_head(&format!("refs/heads/{}", branch)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to '{}' branch: {}", branch, e), + ) + })?; + + println!("Switched to '{}' branch", branch); + } } else { println!("Cloned default branch only"); } From f95ff9d76593a2d241881dae9e3a244838d12996 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 27 Oct 2024 23:57:56 +0530 Subject: [PATCH 12/55] tag working , now need to add that to the embedArgs --- macros/src/lib.rs | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index eaf04c8..19d428f 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,7 +2,7 @@ use common_path::common_path; use derive_syn_parse::Parse; -use git2::{FetchOptions, Oid, RemoteCallbacks, Repository}; +use git2::{FetchOptions, ObjectType, Oid, RemoteCallbacks, Repository}; use once_cell::sync::Lazy; use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; @@ -1052,7 +1052,8 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - git_url.value().as_str(), &crate_root, None, - Some("4a9108d93525db4be232c5d03998fd7dadcd65c9"), + Some("b1f8b7b6349d71293d3f4d67e23ddb4374812100"), + None, )?; let file_path = args.file_path.value().to_string(); @@ -1338,6 +1339,7 @@ fn clone_repo( project_root: &PathBuf, branch_name: Option<&str>, commit_hash: Option<&str>, + tag_name: Option<&str>, ) -> Result { // Create a temporary directory within the project root let temp_dir = TempDir::new_in(project_root).map_err(|e| { @@ -1374,12 +1376,12 @@ fn clone_repo( println!("Repository cloned successfully"); println!("Repository cloned to: {}", temp_dir.path().display()); - if branch_name.is_some() || commit_hash.is_some() { + if branch_name.is_some() || commit_hash.is_some() || tag_name.is_some() { // Set up fetch options let mut fetch_opts = FetchOptions::new(); fetch_opts.remote_callbacks(callbacks); - // Fetch all branches + // Fetch all branches and tags let mut remote = repo.find_remote("origin").map_err(|e| { Error::new( Span::call_site(), @@ -1388,15 +1390,45 @@ fn clone_repo( })?; remote - .fetch(&["refs/heads/*:refs/heads/*"], Some(&mut fetch_opts), None) + .fetch( + &["refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"], + Some(&mut fetch_opts), + None, + ) .map_err(|e| { Error::new( Span::call_site(), - format!("Failed to fetch all branches: {}", e), + format!("Failed to fetch all branches and tags: {}", e), ) })?; - if let Some(commit) = commit_hash { + if let Some(tag) = tag_name { + // Checkout the specific tag + let (object, reference) = + repo.revparse_ext(&format!("refs/tags/{}", tag)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find tag '{}': {}", tag, e), + ) + })?; + + repo.checkout_tree(&object, None).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to checkout tag '{}': {}", tag, e), + ) + })?; + + repo.set_head_detached(object.id()).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to tag '{}': {}", tag, e), + ) + })?; + + println!("Checked out tag '{}'", tag); + } else if let Some(commit) = commit_hash { // Checkout the specific commit let oid = Oid::from_str(commit).map_err(|e| { Error::new( @@ -1462,7 +1494,6 @@ fn clone_repo( Ok(temp_dir.into_path()) } - fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { let full_path = crate_root.join(file_path); println!( From 6856250d65e1808c0dbf02fdcb98f921f6494193 Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 28 Oct 2024 00:37:04 +0530 Subject: [PATCH 13/55] added commit , branch , tag in embed args , along with validation that , either one of commit , branch , tag can only be passed with git , also added validation check that none of tag , commit , branch be passed without specifying git argument --- macros/src/lib.rs | 130 +++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 19d428f..f2f88a6 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -469,7 +469,7 @@ fn export_internal( /// /// Which will expand to the `my_example` item in `path/to/file.rs` being embedded in a rust /// doc example marked with `ignore`. If you want to have your example actually run in rust -/// docs as well, you should use [`docify::embed_run!(..)`](`macro@embed_run`). +/// docs as well, you should use [`docify::embed_run!(..)`](`macro@embed_run`) instead. /// /// ### Arguments /// - `source_path`: the file path (relative to the current crate root) that contains the item @@ -502,7 +502,6 @@ fn export_internal( /// ```ignore /// /// Here is a cool example module: /// #[doc = docify::embed!("examples/my_example.rs")] -/// struct DocumentedItem /// ``` /// /// You are also free to embed multiple examples in the same set of doc comments: @@ -552,79 +551,103 @@ pub fn embed_run(tokens: TokenStream) -> TokenStream { mod kw { syn::custom_keyword!(git); syn::custom_keyword!(path); + syn::custom_keyword!(branch); + syn::custom_keyword!(commit); + syn::custom_keyword!(tag); syn::custom_keyword!(item); } struct EmbedArgs { git_url: Option, file_path: LitStr, + branch_name: Option, + commit_hash: Option, + tag_name: Option, item_ident: Option, } - impl Parse for EmbedArgs { fn parse(input: ParseStream) -> Result { - println!("impl Parse for EmbedArgs: ----> Starting parse function"); let mut git_url = None; let mut file_path = None; + let mut branch_name = None; + let mut commit_hash = None; + let mut tag_name = None; let mut item_ident = None; while !input.is_empty() { - println!("impl Parse for EmbedArgs: ----> Entering loop iteration"); let lookahead = input.lookahead1(); + if lookahead.peek(kw::git) { - println!("impl Parse for EmbedArgs: ----> Parsing git URL"); - input.parse::()?; - input.parse::()?; - git_url = Some(input.parse::()?); - println!( - "impl Parse for EmbedArgs: ----> Git URL parsed: {}", - git_url.as_ref().map(LitStr::value).unwrap_or_default() - ); + let _: kw::git = input.parse()?; + let _: Token![=] = input.parse()?; + git_url = Some(input.parse()?); } else if lookahead.peek(kw::path) { - println!("impl Parse for EmbedArgs: ----> Parsing file path"); - input.parse::()?; - input.parse::()?; - file_path = Some(input.parse::()?); - println!( - "impl Parse for EmbedArgs: ----> File path parsed: {}", - file_path.as_ref().map(LitStr::value).unwrap_or_default() - ); + let _: kw::path = input.parse()?; + let _: Token![=] = input.parse()?; + file_path = Some(input.parse()?); + } else if lookahead.peek(kw::branch) { + let _: kw::branch = input.parse()?; + let _: Token![=] = input.parse()?; + branch_name = Some(input.parse()?); + } else if lookahead.peek(kw::commit) { + let _: kw::commit = input.parse()?; + let _: Token![=] = input.parse()?; + commit_hash = Some(input.parse()?); + } else if lookahead.peek(kw::tag) { + let _: kw::tag = input.parse()?; + let _: Token![=] = input.parse()?; + tag_name = Some(input.parse()?); } else if lookahead.peek(kw::item) { - println!("impl Parse for EmbedArgs: ----> Parsing item identifier"); - input.parse::()?; - input.parse::()?; - item_ident = Some(input.parse::()?); - println!( - "impl Parse for EmbedArgs: ----> Item identifier parsed: {:?}", - item_ident - ); + let _: kw::item = input.parse()?; + let _: Token![=] = input.parse()?; + item_ident = Some(input.parse()?); } else { - println!("impl Parse for EmbedArgs: ----> Encountered unexpected token"); return Err(lookahead.error()); } + // Parse optional comma after each parameter if !input.is_empty() { - println!("impl Parse for EmbedArgs: ----> Parsing comma separator"); - input.parse::()?; + let _: Token![,] = input.parse()?; } } - println!("impl Parse for EmbedArgs: ----> Finished parsing all arguments"); - let file_path = file_path.ok_or_else(|| input.error("expected `path` argument"))?; - println!( - "impl Parse for EmbedArgs: ----> Final file path: {}", - file_path.value() - ); + // Validate required parameters + let file_path = + file_path.ok_or_else(|| Error::new(Span::call_site(), "path parameter is required"))?; + + // Validate that only one of branch, commit, or tag is specified + let ref_count = [&branch_name, &commit_hash, &tag_name] + .iter() + .filter(|&&x| x.is_some()) + .count(); + + if ref_count > 1 { + return Err(Error::new( + Span::call_site(), + "Only one of branch, commit, or tag can be specified", + )); + } + + // Validate that branch, commit, or tag are only used with git parameter + if git_url.is_none() + && (branch_name.is_some() || commit_hash.is_some() || tag_name.is_some()) + { + return Err(Error::new( + Span::call_site(), + "branch, commit, or tag can only be used when git parameter is specified", + )); + } - println!("impl Parse for EmbedArgs: ----> Returning parsed EmbedArgs"); Ok(EmbedArgs { git_url, file_path, + branch_name, + commit_hash, + tag_name, item_ident, }) } } - impl ToTokens for EmbedArgs { fn to_tokens(&self, tokens: &mut TokenStream2) { tokens.extend(self.file_path.to_token_stream()); @@ -1051,9 +1074,9 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let repo_path = clone_repo( git_url.value().as_str(), &crate_root, - None, - Some("b1f8b7b6349d71293d3f4d67e23ddb4374812100"), - None, + args.branch_name.as_ref().map(|b| b.value()), + args.commit_hash.as_ref().map(|c| c.value()), + args.tag_name.as_ref().map(|t| t.value()), )?; let file_path = args.file_path.value().to_string(); @@ -1333,15 +1356,13 @@ fn compile_markdown_dir, P2: AsRef>( } Ok(()) } - fn clone_repo( git_url: &str, project_root: &PathBuf, - branch_name: Option<&str>, - commit_hash: Option<&str>, - tag_name: Option<&str>, + branch_name: Option, + commit_hash: Option, + tag_name: Option, ) -> Result { - // Create a temporary directory within the project root let temp_dir = TempDir::new_in(project_root).map_err(|e| { Error::new( Span::call_site(), @@ -1354,7 +1375,6 @@ fn clone_repo( temp_dir.path().display() ); - // Set up remote callbacks let mut callbacks = RemoteCallbacks::new(); callbacks.transfer_progress(|progress| { println!( @@ -1365,7 +1385,6 @@ fn clone_repo( true }); - // Clone the repository let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { Error::new( Span::call_site(), @@ -1377,11 +1396,9 @@ fn clone_repo( println!("Repository cloned to: {}", temp_dir.path().display()); if branch_name.is_some() || commit_hash.is_some() || tag_name.is_some() { - // Set up fetch options let mut fetch_opts = FetchOptions::new(); fetch_opts.remote_callbacks(callbacks); - // Fetch all branches and tags let mut remote = repo.find_remote("origin").map_err(|e| { Error::new( Span::call_site(), @@ -1402,8 +1419,7 @@ fn clone_repo( ) })?; - if let Some(tag) = tag_name { - // Checkout the specific tag + if let Some(tag) = tag_name.as_deref() { let (object, reference) = repo.revparse_ext(&format!("refs/tags/{}", tag)) .map_err(|e| { @@ -1428,8 +1444,7 @@ fn clone_repo( })?; println!("Checked out tag '{}'", tag); - } else if let Some(commit) = commit_hash { - // Checkout the specific commit + } else if let Some(commit) = commit_hash.as_deref() { let oid = Oid::from_str(commit).map_err(|e| { Error::new( Span::call_site(), @@ -1460,8 +1475,7 @@ fn clone_repo( })?; println!("Checked out commit '{}'", commit); - } else if let Some(branch) = branch_name { - // Switch to the specified branch + } else if let Some(branch) = branch_name.as_deref() { let (object, reference) = repo.revparse_ext(&format!("origin/{}", branch)) .map_err(|e| { From f4f545eb73c7b12ae8813717e105584e4c693bc0 Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 28 Oct 2024 01:14:39 +0530 Subject: [PATCH 14/55] internet check added --- macros/src/lib.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index f2f88a6..64bb33b 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -9,6 +9,7 @@ use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, ToTokens}; use regex::Regex; use sha2::{Digest, Sha256}; +use std::net::TcpStream; use std::{ cmp::min, collections::HashMap, @@ -1508,6 +1509,7 @@ fn clone_repo( Ok(temp_dir.into_path()) } + fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { let full_path = crate_root.join(file_path); println!( @@ -1530,10 +1532,11 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul let snippet_path = generate_snippet_path(&snippets_dir, file_path, item_ident); println!("Snippet path: {}", snippet_path.display()); - let is_docs_rs = std::env::var("DOCS_RS").is_ok(); + // Check internet connectivity by attempting to connect to a reliable host + let has_internet = TcpStream::connect("8.8.8.8:53").is_ok(); - if is_docs_rs { - println!("Running on docs.rs, reading existing snippet"); + if !has_internet { + println!("No internet connection, reading existing snippet"); fs::read_to_string(&snippet_path).map_err(|e| { Error::new( Span::call_site(), @@ -1541,7 +1544,7 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul ) }) } else { - println!("Local development, checking if snippet needs updating"); + println!("Internet connection available, checking if snippet needs updating"); let existing_content = fs::read_to_string(&snippet_path).ok(); let new_content = extract_item_from_file(&full_path, item_ident)?; From f3787809e70a0928e895cd81e17814f9ae91573f Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 28 Oct 2024 22:23:34 +0530 Subject: [PATCH 15/55] added utility function for checking internet --- macros/src/lib.rs | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 64bb33b..c3f680f 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -10,6 +10,7 @@ use quote::{quote, ToTokens}; use regex::Regex; use sha2::{Digest, Sha256}; use std::net::TcpStream; +use std::time::Duration; use std::{ cmp::min, collections::HashMap, @@ -1050,7 +1051,9 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ); // Check if the file_path starts with "../" and git_url is not provided - if args.file_path.value().starts_with("../") && args.git_url.is_none() { + if (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("/")) + && args.git_url.is_none() + { return Err(Error::new( args.file_path.span(), "You can only embed files which are present in the current crate. For any other files, please provide the git_url to embed." @@ -1533,7 +1536,16 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul println!("Snippet path: {}", snippet_path.display()); // Check internet connectivity by attempting to connect to a reliable host - let has_internet = TcpStream::connect("8.8.8.8:53").is_ok(); + let has_internet = check_internet_connectivity(); + + println!( + "Internet connectivity: {}", + if has_internet { + "Available" + } else { + "Not available" + } + ); if !has_internet { println!("No internet connection, reading existing snippet"); @@ -1640,6 +1652,31 @@ fn hash_content(content: &str) -> String { result } +/// Checks if there is an active internet connection by attempting to connect to multiple reliable hosts +fn check_internet_connectivity() -> bool { + // List of reliable hosts and ports to try + let hosts = [ + ("8.8.8.8", 53), // Google DNS + ("1.1.1.1", 53), // Cloudflare DNS + ("208.67.222.222", 53), // OpenDNS + ]; + + // Set a timeout for connection attempts + let timeout = Duration::from_secs(1); + + for &(host, port) in hosts.iter() { + if let Ok(stream) = TcpStream::connect((host, port)) { + // Set the timeout for read/write operations + if stream.set_read_timeout(Some(timeout)).is_ok() + && stream.set_write_timeout(Some(timeout)).is_ok() + { + return true; + } + } + } + + false +} /// Docifies the specified markdown source string fn compile_markdown_source>(source: S) -> Result { let source = source.as_ref(); From 11aebc35d45ac909c634c966dc11c0249e2d7417 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 29 Oct 2024 01:37:53 +0530 Subject: [PATCH 16/55] network check working tested --- macros/src/lib.rs | 73 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index c3f680f..0627727 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -521,7 +521,7 @@ fn export_internal( /// and so they do not need to be run as well in the context where they are being embedded. If /// for whatever reason you _do_ want to also run an embedded example as a doc example, you can /// use [`docify::embed_run!(..)`](`macro@embed_run`) which removes the `ignore` tag from the -/// generated example but otherwise functions exactly like `#[docify::embed!(..)]` in every +/// generated example but otherwise functions exactly like `#[docify::embed!(..)` in every /// way. /// /// Output should match `rustfmt` output exactly. @@ -1074,6 +1074,28 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; if let Some(git_url) = &args.git_url { + let has_internet = check_internet_connectivity(); + + if !has_internet { + if !has_internet { + println!("No internet connection detected. Using cached snippet if available in the snippets dir."); + // Get content from snippet and process it + let content = manage_snippet( + &crate_root, + &args.file_path.value(), + args.item_ident + .as_ref() + .map(|i| i.to_string()) + .unwrap_or_default() + .as_str(), + )?; + + // Fix indentation and process the content + let fixed = fix_indentation(&content); + return Ok(into_example(&fixed, lang)); + } + } + println!("Detected git-based embedding"); let repo_path = clone_repo( git_url.value().as_str(), @@ -1514,17 +1536,10 @@ fn clone_repo( } fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { - let full_path = crate_root.join(file_path); - println!( - "inside manage_snippet ----> Full path to crate root: {}", - crate_root.display() - ); - println!( - "inside manage_snippet ----> Full path to file: {}", - full_path.display() - ); - let snippets_dir = crate_root.join(".snippets"); + println!("Snippets directory: {}", snippets_dir.display()); + + // Ensure snippets directory exists fs::create_dir_all(&snippets_dir).map_err(|e| { Error::new( Span::call_site(), @@ -1535,9 +1550,7 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul let snippet_path = generate_snippet_path(&snippets_dir, file_path, item_ident); println!("Snippet path: {}", snippet_path.display()); - // Check internet connectivity by attempting to connect to a reliable host let has_internet = check_internet_connectivity(); - println!( "Internet connectivity: {}", if has_internet { @@ -1548,20 +1561,36 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul ); if !has_internet { - println!("No internet connection, reading existing snippet"); - fs::read_to_string(&snippet_path).map_err(|e| { - Error::new( + println!("No internet connection, attempting to read from cached snippet"); + // Try to read from the snippet file + match fs::read_to_string(&snippet_path) { + Ok(content) => { + println!("Successfully read content from cached snippet"); + Ok(content) + } + Err(e) => Err(Error::new( Span::call_site(), - format!("Failed to read snippet file: {}", e), - ) - }) + format!( + "No internet connection and failed to read cached snippet at {}: {}. + Please ensure you have internet connectivity for the first run.", + snippet_path.display(), + e + ), + )), + } } else { - println!("Internet connection available, checking if snippet needs updating"); + // Internet is available, proceed with full path checking and content updating + let full_path = crate_root.join(file_path); + println!( + "Internet available, checking file at: {}", + full_path.display() + ); + let existing_content = fs::read_to_string(&snippet_path).ok(); let new_content = extract_item_from_file(&full_path, item_ident)?; if existing_content.as_ref().map(|c| hash_content(c)) != Some(hash_content(&new_content)) { - println!("inside manage_snippet ----> Updating snippet file"); + println!("Updating snippet file with new content"); fs::write(&snippet_path, &new_content).map_err(|e| { Error::new( Span::call_site(), @@ -1569,7 +1598,7 @@ fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Resul ) })?; } else { - println!("inside manage_snippet ----> Snippet is up to date"); + println!("Snippet is up to date"); } Ok(new_content) From b1a88b5f4be5747e17ebd788dbb847bc49fcdc17 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 29 Oct 2024 02:25:17 +0530 Subject: [PATCH 17/55] positional arguments working --- macros/src/lib.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 0627727..f191b45 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -558,7 +558,6 @@ mod kw { syn::custom_keyword!(tag); syn::custom_keyword!(item); } - struct EmbedArgs { git_url: Option, file_path: LitStr, @@ -567,8 +566,30 @@ struct EmbedArgs { tag_name: Option, item_ident: Option, } + impl Parse for EmbedArgs { fn parse(input: ParseStream) -> Result { + // First try to parse as positional arguments + if !input.peek(kw::git) && !input.peek(kw::path) { + let file_path: LitStr = input.parse()?; + let item_ident = if input.peek(Token![,]) { + input.parse::()?; + Some(input.parse()?) + } else { + None + }; + + return Ok(EmbedArgs { + git_url: None, + file_path, + branch_name: None, + commit_hash: None, + tag_name: None, + item_ident, + }); + } + + // If not positional, parse as named arguments let mut git_url = None; let mut file_path = None; let mut branch_name = None; @@ -607,7 +628,6 @@ impl Parse for EmbedArgs { return Err(lookahead.error()); } - // Parse optional comma after each parameter if !input.is_empty() { let _: Token![,] = input.parse()?; } @@ -650,6 +670,7 @@ impl Parse for EmbedArgs { }) } } + impl ToTokens for EmbedArgs { fn to_tokens(&self, tokens: &mut TokenStream2) { tokens.extend(self.file_path.to_token_stream()); From 66c171ea162f0fc20e34e44755310344ced0aee9 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 29 Oct 2024 03:07:50 +0530 Subject: [PATCH 18/55] added validation on first positional argument that it can't be a url , it must be a file path --- macros/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index f191b45..7bd8883 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -572,6 +572,16 @@ impl Parse for EmbedArgs { // First try to parse as positional arguments if !input.peek(kw::git) && !input.peek(kw::path) { let file_path: LitStr = input.parse()?; + + // Validate that file_path is not a URL + if file_path.value().starts_with("http://") || file_path.value().starts_with("https://") + { + return Err(Error::new( + file_path.span(), + "First positional argument must be a file path, not a URL. For git URLs, use named arguments: git = \"url\", path = \"file_path\"", + )); + } + let item_ident = if input.peek(Token![,]) { input.parse::()?; Some(input.parse()?) From 45f219dde640505b30bdb5277d8f6744815005e8 Mon Sep 17 00:00:00 2001 From: Prakash Date: Wed, 30 Oct 2024 23:37:33 +0530 Subject: [PATCH 19/55] tmp dir clean up but need to persist it till the lifecycle of the entire crate process --- macros/src/lib.rs | 450 ++++++++++++++++++++++++++-------------------- 1 file changed, 253 insertions(+), 197 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 7bd8883..3083537 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -10,6 +10,7 @@ use quote::{quote, ToTokens}; use regex::Regex; use sha2::{Digest, Sha256}; use std::net::TcpStream; +use std::sync::Mutex; use std::time::Duration; use std::{ cmp::min, @@ -29,7 +30,7 @@ use syn::{ AttrStyle, Attribute, Error, File, Ident, ImplItem, Item, LitStr, Meta, Result, Token, TraitItem, }; -use tempfile::TempDir; +use tempfile::{Builder, TempDir}; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use toml::{Table, Value}; use walkdir::WalkDir; @@ -1128,7 +1129,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } println!("Detected git-based embedding"); - let repo_path = clone_repo( + let temp_dir = clone_repo( git_url.value().as_str(), &crate_root, args.branch_name.as_ref().map(|b| b.value()), @@ -1137,9 +1138,12 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - )?; let file_path = args.file_path.value().to_string(); - let full_path = repo_path.join(&file_path).to_string_lossy().into_owned(); + let full_path = temp_dir + .path() + .join(&file_path) + .to_string_lossy() + .into_owned(); println!("embed_internal_str ----> Full path: {}", full_path); - let snippet_content = match &args.item_ident { Some(ident) => { println!("Generating snippet for item: {}", ident); @@ -1147,7 +1151,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } None => { println!("No specific item requested, using entire file content"); - fs::read_to_string(repo_path.join(&file_path)).map_err(|e| { + fs::read_to_string(temp_dir.path().join(&file_path)).map_err(|e| { Error::new(Span::call_site(), format!("Failed to read file: {}", e)) })? } @@ -1229,203 +1233,69 @@ fn embed_internal(tokens: impl Into, lang: MarkdownLanguage) -> Re Ok(quote!(#output)) } -/// Used to parse args for [`macro@compile_markdown`]. -#[derive(Parse)] -struct CompileMarkdownArgs { - input: LitStr, - #[prefix(Option as comma)] - #[parse_if(comma.is_some())] - output: Option, -} +// // Cache for storing repository clones +// static REPO_CACHE: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); + +// // Generate a unique cache key for a repository +// fn generate_cache_key( +// git_url: &str, +// branch: Option<&str>, +// commit: Option<&str>, +// tag: Option<&str>, +// ) -> String { +// format!( +// "{}:{}:{}:{}", +// git_url, +// branch.unwrap_or("master"), +// commit.unwrap_or("none"), +// tag.unwrap_or("none") +// ) +// } + +// fn get_or_clone_repo( +// git_url: &str, +// project_root: &PathBuf, +// branch_name: Option, +// commit_hash: Option, +// tag_name: Option, +// ) -> Result { +// let cache_key = generate_cache_key( +// git_url, +// branch_name.as_deref(), +// commit_hash.as_deref(), +// tag_name.as_deref(), +// ); + +// let mut cache = REPO_CACHE.lock().unwrap(); + +// if !cache.contains_key(&cache_key) { +// println!("Cache miss for {}, cloning repository...", git_url); +// let temp_dir = clone_repo(git_url, project_root, branch_name, commit_hash, tag_name)?; +// cache.insert(cache_key.clone(), temp_dir); +// } else { +// println!("Cache hit for {}, using existing clone", git_url); +// } + +// Ok(cache.get(&cache_key).unwrap().path().to_path_buf()) +// } -/// Internal implementation behind [`macro@compile_markdown`]. -fn compile_markdown_internal(tokens: impl Into) -> Result { - let args = parse2::(tokens.into())?; - if args.input.value().is_empty() { - return Err(Error::new(args.input.span(), "Input path cannot be blank!")); - } - let input_path = std::path::PathBuf::from(&args.input.value()); - // return blank result if we can't properly resolve `caller_crate_root` - let Some(root) = caller_crate_root() else { - return Ok(quote!()); - }; - let input_path = root.join(input_path); - if !input_path.exists() { - return Err(Error::new( - args.input.span(), - format!( - "Could not read the specified path '{}'.", - input_path.display(), - ), - )); - } - if let Some(output) = args.output { - if output.value().is_empty() { - return Err(Error::new( - output.span(), - "If specified, output path cannot be blank!", - )); - } - let output = root.join(output.value()); - if input_path.is_dir() { - compile_markdown_dir(input_path, format!("{}", output.display()))?; - } else { - if cfg!(not(test)) { - write_green(DOCIFYING); - println!( - "{} {} {}", - prettify_path(&input_path).display(), - "=>", // TODO: fancy arrow - prettify_path(&output).display(), - ); - } - let Ok(source) = fs::read_to_string(&input_path) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to read markdown file at '{}'", input_path.display()), - )); - }; - let compiled = compile_markdown_source(source.as_str())?; - let Ok(_) = overwrite_file(&output, &compiled) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to write to '{}'", output.display()), - )); - }; - } - Ok(quote!()) - } else { - if input_path.is_dir() { - return Err(Error::new( - args.input.span(), - "Only individual files are supported with no output path, you specified a directory." - )); - } - let Ok(source) = fs::read_to_string(&input_path) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to read markdown file at '{}'", input_path.display()), - )); - }; - let compiled = compile_markdown_source(source.as_str())?; - Ok(quote!(#compiled)) - } -} - -/// Takes in a `path` and re-writes it as a subpath in `target_dir`. -fn transpose_subpath, P2: AsRef, P3: AsRef>( - input_dir: P1, - path: P2, - target_dir: P3, -) -> PathBuf { - let prefix = common_path(input_dir, &path).unwrap(); - Path::join( - target_dir.as_ref(), - path.as_ref() - .components() - .skip(prefix.components().collect::>().len()) - .collect::(), - ) -} - -/// Overwrites or creates a file at the specified path and populates it with the specified -/// data. Will only overwrite the file if the data is different from what is already there. -fn overwrite_file, D: AsRef<[u8]>>(path: P, data: D) -> std::io::Result<()> { - if path.as_ref().exists() { - if let Ok(existing) = fs::read(path.as_ref()) { - if existing == data.as_ref() { - return Ok(()); - } - } - } - let mut f = OpenOptions::new() - .write(true) - .truncate(true) - .create(true) - .open(path)?; - f.write_all(data.as_ref())?; - f.flush()?; - Ok(()) -} - -/// Docifies a directory of markdown files -fn compile_markdown_dir, P2: AsRef>( - input_dir: P1, - output_dir: P2, -) -> Result<()> { - // recursively walk all files in output_dir - for entry in WalkDir::new(&input_dir) - .into_iter() - .filter_map(std::result::Result::ok) - .filter(|e| { - if !e.file_type().is_file() && !e.file_type().is_symlink() { - return false; - } - let Some(ext) = e.path().extension() else { - return false; - }; - if ext.eq_ignore_ascii_case("md") { - return true; - } - false - }) - { - let src_path = entry.path(); - let dest_path = transpose_subpath(&input_dir, &src_path, &output_dir); - if cfg!(not(test)) { - write_green(DOCIFYING); - println!( - "{} {} {}", - prettify_path(&src_path).display(), - "=>", // TODO: fancy arrow - prettify_path(&dest_path).display(), - ); - } - if let Some(parent) = dest_path.parent() { - let Ok(_) = fs::create_dir_all(parent) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to create output directory '{}'", parent.display()), - )); - }; - } - let Ok(source) = fs::read_to_string(src_path) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to read markdown file at '{}'", src_path.display()), - )); - }; - let compiled = compile_markdown_source(source.as_str())?; - if let Some(parent) = dest_path.parent() { - let Ok(_) = fs::create_dir_all(parent) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to create directory '{}'", parent.display()), - )); - }; - } - let Ok(_) = overwrite_file(&dest_path, &compiled) else { - return Err(Error::new( - Span::call_site(), - format!("Failed to write to '{}'", dest_path.display()), - )); - }; - } - Ok(()) -} fn clone_repo( git_url: &str, - project_root: &PathBuf, + _project_root: &PathBuf, branch_name: Option, commit_hash: Option, tag_name: Option, -) -> Result { - let temp_dir = TempDir::new_in(project_root).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create temporary directory: {}", e), - ) - })?; +) -> Result { + let temp_dir = Builder::new() + .prefix("docify-") + .rand_bytes(5) + .tempdir() + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create temp directory: {}", e), + ) + })?; println!( "Temporary directory created at: {}", @@ -1563,7 +1433,7 @@ fn clone_repo( println!("Cloned default branch only"); } - Ok(temp_dir.into_path()) + Ok(temp_dir) } fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { @@ -1737,6 +1607,192 @@ fn check_internet_connectivity() -> bool { false } + +/// Used to parse args for [`macro@compile_markdown`]. +#[derive(Parse)] +struct CompileMarkdownArgs { + input: LitStr, + #[prefix(Option as comma)] + #[parse_if(comma.is_some())] + output: Option, +} + +/// Internal implementation behind [`macro@compile_markdown`]. +fn compile_markdown_internal(tokens: impl Into) -> Result { + let args = parse2::(tokens.into())?; + if args.input.value().is_empty() { + return Err(Error::new(args.input.span(), "Input path cannot be blank!")); + } + let input_path = std::path::PathBuf::from(&args.input.value()); + // return blank result if we can't properly resolve `caller_crate_root` + let Some(root) = caller_crate_root() else { + return Ok(quote!()); + }; + let input_path = root.join(input_path); + if !input_path.exists() { + return Err(Error::new( + args.input.span(), + format!( + "Could not read the specified path '{}'.", + input_path.display(), + ), + )); + } + if let Some(output) = args.output { + if output.value().is_empty() { + return Err(Error::new( + output.span(), + "If specified, output path cannot be blank!", + )); + } + let output = root.join(output.value()); + if input_path.is_dir() { + compile_markdown_dir(input_path, format!("{}", output.display()))?; + } else { + if cfg!(not(test)) { + write_green(DOCIFYING); + println!( + "{} {} {}", + prettify_path(&input_path).display(), + "=>", // TODO: fancy arrow + prettify_path(&output).display(), + ); + } + let Ok(source) = fs::read_to_string(&input_path) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to read markdown file at '{}'", input_path.display()), + )); + }; + let compiled = compile_markdown_source(source.as_str())?; + let Ok(_) = overwrite_file(&output, &compiled) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to write to '{}'", output.display()), + )); + }; + } + Ok(quote!()) + } else { + if input_path.is_dir() { + return Err(Error::new( + args.input.span(), + "Only individual files are supported with no output path, you specified a directory." + )); + } + let Ok(source) = fs::read_to_string(&input_path) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to read markdown file at '{}'", input_path.display()), + )); + }; + let compiled = compile_markdown_source(source.as_str())?; + Ok(quote!(#compiled)) + } +} + +/// Takes in a `path` and re-writes it as a subpath in `target_dir`. +fn transpose_subpath, P2: AsRef, P3: AsRef>( + input_dir: P1, + path: P2, + target_dir: P3, +) -> PathBuf { + let prefix = common_path(input_dir, &path).unwrap(); + Path::join( + target_dir.as_ref(), + path.as_ref() + .components() + .skip(prefix.components().collect::>().len()) + .collect::(), + ) +} + +/// Overwrites or creates a file at the specified path and populates it with the specified +/// data. Will only overwrite the file if the data is different from what is already there. +fn overwrite_file, D: AsRef<[u8]>>(path: P, data: D) -> std::io::Result<()> { + if path.as_ref().exists() { + if let Ok(existing) = fs::read(path.as_ref()) { + if existing == data.as_ref() { + return Ok(()); + } + } + } + let mut f = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(path)?; + f.write_all(data.as_ref())?; + f.flush()?; + Ok(()) +} + +/// Docifies a directory of markdown files +fn compile_markdown_dir, P2: AsRef>( + input_dir: P1, + output_dir: P2, +) -> Result<()> { + // recursively walk all files in output_dir + for entry in WalkDir::new(&input_dir) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|e| { + if !e.file_type().is_file() && !e.file_type().is_symlink() { + return false; + } + let Some(ext) = e.path().extension() else { + return false; + }; + if ext.eq_ignore_ascii_case("md") { + return true; + } + false + }) + { + let src_path = entry.path(); + let dest_path = transpose_subpath(&input_dir, &src_path, &output_dir); + if cfg!(not(test)) { + write_green(DOCIFYING); + println!( + "{} {} {}", + prettify_path(&src_path).display(), + "=>", // TODO: fancy arrow + prettify_path(&dest_path).display(), + ); + } + if let Some(parent) = dest_path.parent() { + let Ok(_) = fs::create_dir_all(parent) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to create output directory '{}'", parent.display()), + )); + }; + } + let Ok(source) = fs::read_to_string(src_path) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to read markdown file at '{}'", src_path.display()), + )); + }; + let compiled = compile_markdown_source(source.as_str())?; + if let Some(parent) = dest_path.parent() { + let Ok(_) = fs::create_dir_all(parent) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to create directory '{}'", parent.display()), + )); + }; + } + let Ok(_) = overwrite_file(&dest_path, &compiled) else { + return Err(Error::new( + Span::call_site(), + format!("Failed to write to '{}'", dest_path.display()), + )); + }; + } + Ok(()) +} + /// Docifies the specified markdown source string fn compile_markdown_source>(source: S) -> Result { let source = source.as_ref(); From 62f01fdf37a6fe223407ef9eb94b6c8ad4f1459b Mon Sep 17 00:00:00 2001 From: Prakash Date: Thu, 31 Oct 2024 02:11:25 +0530 Subject: [PATCH 20/55] git dir cache key --- macros/src/lib.rs | 101 ++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 53 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 3083537..ebefac8 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1129,20 +1129,17 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } println!("Detected git-based embedding"); - let temp_dir = clone_repo( + println!("Detected git-based embedding"); + let repo_path = get_or_clone_repo( git_url.value().as_str(), &crate_root, args.branch_name.as_ref().map(|b| b.value()), args.commit_hash.as_ref().map(|c| c.value()), args.tag_name.as_ref().map(|t| t.value()), )?; - let file_path = args.file_path.value().to_string(); - let full_path = temp_dir - .path() - .join(&file_path) - .to_string_lossy() - .into_owned(); + let file_path = args.file_path.value().to_string(); + let full_path = repo_path.join(&file_path).to_string_lossy().into_owned(); println!("embed_internal_str ----> Full path: {}", full_path); let snippet_content = match &args.item_ident { Some(ident) => { @@ -1151,7 +1148,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } None => { println!("No specific item requested, using entire file content"); - fs::read_to_string(temp_dir.path().join(&file_path)).map_err(|e| { + fs::read_to_string(repo_path.join(&file_path)).map_err(|e| { Error::new(Span::call_site(), format!("Failed to read file: {}", e)) })? } @@ -1233,52 +1230,50 @@ fn embed_internal(tokens: impl Into, lang: MarkdownLanguage) -> Re Ok(quote!(#output)) } -// // Cache for storing repository clones -// static REPO_CACHE: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); - -// // Generate a unique cache key for a repository -// fn generate_cache_key( -// git_url: &str, -// branch: Option<&str>, -// commit: Option<&str>, -// tag: Option<&str>, -// ) -> String { -// format!( -// "{}:{}:{}:{}", -// git_url, -// branch.unwrap_or("master"), -// commit.unwrap_or("none"), -// tag.unwrap_or("none") -// ) -// } - -// fn get_or_clone_repo( -// git_url: &str, -// project_root: &PathBuf, -// branch_name: Option, -// commit_hash: Option, -// tag_name: Option, -// ) -> Result { -// let cache_key = generate_cache_key( -// git_url, -// branch_name.as_deref(), -// commit_hash.as_deref(), -// tag_name.as_deref(), -// ); - -// let mut cache = REPO_CACHE.lock().unwrap(); - -// if !cache.contains_key(&cache_key) { -// println!("Cache miss for {}, cloning repository...", git_url); -// let temp_dir = clone_repo(git_url, project_root, branch_name, commit_hash, tag_name)?; -// cache.insert(cache_key.clone(), temp_dir); -// } else { -// println!("Cache hit for {}, using existing clone", git_url); -// } - -// Ok(cache.get(&cache_key).unwrap().path().to_path_buf()) -// } +// Cache for storing repository clones +static REPO_CACHE: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); +// Generate a unique cache key for a repository +fn generate_cache_key( + git_url: &str, + branch: Option<&str>, + commit: Option<&str>, + tag: Option<&str>, +) -> String { + format!( + "{}:{}:{}:{}", + git_url, + branch.unwrap_or("master"), + commit.unwrap_or("none"), + tag.unwrap_or("none") + ) +} +fn get_or_clone_repo( + git_url: &str, + project_root: &PathBuf, + branch_name: Option, + commit_hash: Option, + tag_name: Option, +) -> Result { + let cache_key = generate_cache_key( + git_url, + branch_name.as_deref(), + commit_hash.as_deref(), + tag_name.as_deref(), + ); + + let mut cache = REPO_CACHE.lock().unwrap(); + + if !cache.contains_key(&cache_key) { + println!("Cache miss for {}, cloning repository...", git_url); + let temp_dir = clone_repo(git_url, project_root, branch_name, commit_hash, tag_name)?; + cache.insert(cache_key.clone(), temp_dir); + } else { + println!("Cache hit for {}, using existing clone", git_url); + } + + Ok(cache.get(&cache_key).unwrap().path().to_path_buf()) +} fn clone_repo( git_url: &str, _project_root: &PathBuf, From 1b033e5187ed8381c055d4ac2b5f36c379f70760 Mon Sep 17 00:00:00 2001 From: Prakash Date: Thu, 31 Oct 2024 09:11:32 +0530 Subject: [PATCH 21/55] changed "=" to ":" testing of all cases remaining --- macros/src/lib.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index ebefac8..d684a14 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -579,7 +579,7 @@ impl Parse for EmbedArgs { { return Err(Error::new( file_path.span(), - "First positional argument must be a file path, not a URL. For git URLs, use named arguments: git = \"url\", path = \"file_path\"", + "First positional argument must be a file path, not a URL. For git URLs, use named arguments: git: \"url\", path: \"file_path\"", )); } @@ -613,27 +613,27 @@ impl Parse for EmbedArgs { if lookahead.peek(kw::git) { let _: kw::git = input.parse()?; - let _: Token![=] = input.parse()?; + let _: Token![:] = input.parse()?; git_url = Some(input.parse()?); } else if lookahead.peek(kw::path) { let _: kw::path = input.parse()?; - let _: Token![=] = input.parse()?; + let _: Token![:] = input.parse()?; file_path = Some(input.parse()?); } else if lookahead.peek(kw::branch) { let _: kw::branch = input.parse()?; - let _: Token![=] = input.parse()?; + let _: Token![:] = input.parse()?; branch_name = Some(input.parse()?); } else if lookahead.peek(kw::commit) { let _: kw::commit = input.parse()?; - let _: Token![=] = input.parse()?; + let _: Token![:] = input.parse()?; commit_hash = Some(input.parse()?); } else if lookahead.peek(kw::tag) { let _: kw::tag = input.parse()?; - let _: Token![=] = input.parse()?; + let _: Token![:] = input.parse()?; tag_name = Some(input.parse()?); } else if lookahead.peek(kw::item) { let _: kw::item = input.parse()?; - let _: Token![=] = input.parse()?; + let _: Token![:] = input.parse()?; item_ident = Some(input.parse()?); } else { return Err(lookahead.error()); From d67cef7ad054393202df2fdf96267622a994d90d Mon Sep 17 00:00:00 2001 From: Prakash Date: Thu, 31 Oct 2024 09:43:50 +0530 Subject: [PATCH 22/55] separate out functions , refactor --- macros/src/lib.rs | 377 +------------------------------------- macros/src/utils/mod.rs | 389 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+), 375 deletions(-) create mode 100644 macros/src/utils/mod.rs diff --git a/macros/src/lib.rs b/macros/src/lib.rs index d684a14..126b74e 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1,5 +1,4 @@ -//! This crate contains the proc macros used by [docify](https://crates.io/crates/docify). - +mod utils; use common_path::common_path; use derive_syn_parse::Parse; use git2::{FetchOptions, ObjectType, Oid, RemoteCallbacks, Repository}; @@ -33,6 +32,7 @@ use syn::{ use tempfile::{Builder, TempDir}; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use toml::{Table, Value}; +use utils::*; use walkdir::WalkDir; fn line_start_position>(source: S, pos: usize) -> usize { @@ -1230,379 +1230,6 @@ fn embed_internal(tokens: impl Into, lang: MarkdownLanguage) -> Re Ok(quote!(#output)) } -// Cache for storing repository clones -static REPO_CACHE: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); - -// Generate a unique cache key for a repository -fn generate_cache_key( - git_url: &str, - branch: Option<&str>, - commit: Option<&str>, - tag: Option<&str>, -) -> String { - format!( - "{}:{}:{}:{}", - git_url, - branch.unwrap_or("master"), - commit.unwrap_or("none"), - tag.unwrap_or("none") - ) -} -fn get_or_clone_repo( - git_url: &str, - project_root: &PathBuf, - branch_name: Option, - commit_hash: Option, - tag_name: Option, -) -> Result { - let cache_key = generate_cache_key( - git_url, - branch_name.as_deref(), - commit_hash.as_deref(), - tag_name.as_deref(), - ); - - let mut cache = REPO_CACHE.lock().unwrap(); - - if !cache.contains_key(&cache_key) { - println!("Cache miss for {}, cloning repository...", git_url); - let temp_dir = clone_repo(git_url, project_root, branch_name, commit_hash, tag_name)?; - cache.insert(cache_key.clone(), temp_dir); - } else { - println!("Cache hit for {}, using existing clone", git_url); - } - - Ok(cache.get(&cache_key).unwrap().path().to_path_buf()) -} -fn clone_repo( - git_url: &str, - _project_root: &PathBuf, - branch_name: Option, - commit_hash: Option, - tag_name: Option, -) -> Result { - let temp_dir = Builder::new() - .prefix("docify-") - .rand_bytes(5) - .tempdir() - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create temp directory: {}", e), - ) - })?; - - println!( - "Temporary directory created at: {}", - temp_dir.path().display() - ); - - let mut callbacks = RemoteCallbacks::new(); - callbacks.transfer_progress(|progress| { - println!( - "Transfer progress: {}/{} objects", - progress.received_objects(), - progress.total_objects() - ); - true - }); - - let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to clone repository: {}", e), - ) - })?; - - println!("Repository cloned successfully"); - println!("Repository cloned to: {}", temp_dir.path().display()); - - if branch_name.is_some() || commit_hash.is_some() || tag_name.is_some() { - let mut fetch_opts = FetchOptions::new(); - fetch_opts.remote_callbacks(callbacks); - - let mut remote = repo.find_remote("origin").map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find remote 'origin': {}", e), - ) - })?; - - remote - .fetch( - &["refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"], - Some(&mut fetch_opts), - None, - ) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to fetch all branches and tags: {}", e), - ) - })?; - - if let Some(tag) = tag_name.as_deref() { - let (object, reference) = - repo.revparse_ext(&format!("refs/tags/{}", tag)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find tag '{}': {}", tag, e), - ) - })?; - - repo.checkout_tree(&object, None).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout tag '{}': {}", tag, e), - ) - })?; - - repo.set_head_detached(object.id()).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to set HEAD to tag '{}': {}", tag, e), - ) - })?; - - println!("Checked out tag '{}'", tag); - } else if let Some(commit) = commit_hash.as_deref() { - let oid = Oid::from_str(commit).map_err(|e| { - Error::new( - Span::call_site(), - format!("Invalid commit hash '{}': {}", commit, e), - ) - })?; - - let commit_obj = repo.find_commit(oid).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find commit '{}': {}", commit, e), - ) - })?; - - repo.set_head_detached(commit_obj.id()).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to set HEAD to commit '{}': {}", commit, e), - ) - })?; - - repo.checkout_head(Some(git2::build::CheckoutBuilder::new().force())) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout commit '{}': {}", commit, e), - ) - })?; - - println!("Checked out commit '{}'", commit); - } else if let Some(branch) = branch_name.as_deref() { - let (object, reference) = - repo.revparse_ext(&format!("origin/{}", branch)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find '{}' branch: {}", branch, e), - ) - })?; - - repo.checkout_tree(&object, None).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout '{}' branch: {}", branch, e), - ) - })?; - - repo.set_head(&format!("refs/heads/{}", branch)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to set HEAD to '{}' branch: {}", branch, e), - ) - })?; - - println!("Switched to '{}' branch", branch); - } - } else { - println!("Cloned default branch only"); - } - - Ok(temp_dir) -} - -fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { - let snippets_dir = crate_root.join(".snippets"); - println!("Snippets directory: {}", snippets_dir.display()); - - // Ensure snippets directory exists - fs::create_dir_all(&snippets_dir).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create .snippets directory: {}", e), - ) - })?; - - let snippet_path = generate_snippet_path(&snippets_dir, file_path, item_ident); - println!("Snippet path: {}", snippet_path.display()); - - let has_internet = check_internet_connectivity(); - println!( - "Internet connectivity: {}", - if has_internet { - "Available" - } else { - "Not available" - } - ); - - if !has_internet { - println!("No internet connection, attempting to read from cached snippet"); - // Try to read from the snippet file - match fs::read_to_string(&snippet_path) { - Ok(content) => { - println!("Successfully read content from cached snippet"); - Ok(content) - } - Err(e) => Err(Error::new( - Span::call_site(), - format!( - "No internet connection and failed to read cached snippet at {}: {}. - Please ensure you have internet connectivity for the first run.", - snippet_path.display(), - e - ), - )), - } - } else { - // Internet is available, proceed with full path checking and content updating - let full_path = crate_root.join(file_path); - println!( - "Internet available, checking file at: {}", - full_path.display() - ); - - let existing_content = fs::read_to_string(&snippet_path).ok(); - let new_content = extract_item_from_file(&full_path, item_ident)?; - - if existing_content.as_ref().map(|c| hash_content(c)) != Some(hash_content(&new_content)) { - println!("Updating snippet file with new content"); - fs::write(&snippet_path, &new_content).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to write snippet file: {}", e), - ) - })?; - } else { - println!("Snippet is up to date"); - } - - Ok(new_content) - } -} - -fn generate_snippet_path(snippets_dir: &Path, file_path: &str, item_ident: &str) -> PathBuf { - println!( - "inside generate_snippet_path ----> Snippets directory: {}", - snippets_dir.display() - ); - println!( - "inside generate_snippet_path ----> File path: {}", - file_path - ); - println!( - "inside generate_snippet_path ----> Item ident: {}", - item_ident - ); - let path = PathBuf::from(file_path); - let file_name = path - .file_name() - .and_then(|f| f.to_str()) - .unwrap_or("unknown"); - snippets_dir.join(format!(".docify-snippet-{}-{}", file_name, item_ident)) -} -fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { - println!( - "inside extract_item_from_file ----> Extracting item '{}' from '{}'", - item_ident, - file_path.display() - ); - - let source_code = fs::read_to_string(file_path).map_err(|e| { - Error::new( - Span::call_site(), - format!( - "Could not read the specified path '{}': {}", - file_path.display(), - e - ), - ) - })?; - - println!( - "inside extract_item_from_file ----> Source code: {}", - source_code - ); - - let mut visitor = ItemVisitor { - search: syn::parse_str(item_ident)?, - results: Vec::new(), - }; - visitor.visit_file(&syn::parse_file(&source_code)?); - println!( - "inside extract_item_from_file ----> Visitor results: {:?}", - visitor.results - ); - if visitor.results.is_empty() { - return Err(Error::new( - Span::call_site(), - format!( - "Could not find docify export item '{}' in '{}'.", - item_ident, - file_path.display() - ), - )); - } - - println!("Successfully extracted item from file"); - let (item, style) = visitor.results.first().unwrap(); - source_excerpt(&source_code, item, *style) -} - -fn hash_content(content: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(content); - let result = format!("{:x}", hasher.finalize()); - println!("Content hash: {}", result); - result -} - -/// Checks if there is an active internet connection by attempting to connect to multiple reliable hosts -fn check_internet_connectivity() -> bool { - // List of reliable hosts and ports to try - let hosts = [ - ("8.8.8.8", 53), // Google DNS - ("1.1.1.1", 53), // Cloudflare DNS - ("208.67.222.222", 53), // OpenDNS - ]; - - // Set a timeout for connection attempts - let timeout = Duration::from_secs(1); - - for &(host, port) in hosts.iter() { - if let Ok(stream) = TcpStream::connect((host, port)) { - // Set the timeout for read/write operations - if stream.set_read_timeout(Some(timeout)).is_ok() - && stream.set_write_timeout(Some(timeout)).is_ok() - { - return true; - } - } - } - - false -} - /// Used to parse args for [`macro@compile_markdown`]. #[derive(Parse)] struct CompileMarkdownArgs { diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs new file mode 100644 index 0000000..a1b767b --- /dev/null +++ b/macros/src/utils/mod.rs @@ -0,0 +1,389 @@ +use git2::{FetchOptions, Oid, RemoteCallbacks, Repository}; +use once_cell::sync::Lazy; +use proc_macro2::Span; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::Duration; +use syn::visit::Visit; +use syn::Error; +use syn::Result; +use tempfile::{Builder, TempDir}; + +use crate::{source_excerpt, ItemVisitor}; + +// Cache for storing repository clones +static REPO_CACHE: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); + +// Generate a unique cache key for a repository +pub fn generate_cache_key( + git_url: &str, + branch: Option<&str>, + commit: Option<&str>, + tag: Option<&str>, +) -> String { + format!( + "{}:{}:{}:{}", + git_url, + branch.unwrap_or("master"), + commit.unwrap_or("none"), + tag.unwrap_or("none") + ) +} +pub fn get_or_clone_repo( + git_url: &str, + project_root: &PathBuf, + branch_name: Option, + commit_hash: Option, + tag_name: Option, +) -> Result { + let cache_key = generate_cache_key( + git_url, + branch_name.as_deref(), + commit_hash.as_deref(), + tag_name.as_deref(), + ); + + let mut cache = REPO_CACHE.lock().unwrap(); + + if !cache.contains_key(&cache_key) { + println!("Cache miss for {}, cloning repository...", git_url); + let temp_dir = clone_repo(git_url, project_root, branch_name, commit_hash, tag_name)?; + cache.insert(cache_key.clone(), temp_dir); + } else { + println!("Cache hit for {}, using existing clone", git_url); + } + + Ok(cache.get(&cache_key).unwrap().path().to_path_buf()) +} +pub fn clone_repo( + git_url: &str, + _project_root: &PathBuf, + branch_name: Option, + commit_hash: Option, + tag_name: Option, +) -> Result { + let temp_dir = Builder::new() + .prefix("docify-") + .rand_bytes(5) + .tempdir() + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create temp directory: {}", e), + ) + })?; + + println!( + "Temporary directory created at: {}", + temp_dir.path().display() + ); + + let mut callbacks = RemoteCallbacks::new(); + callbacks.transfer_progress(|progress| { + println!( + "Transfer progress: {}/{} objects", + progress.received_objects(), + progress.total_objects() + ); + true + }); + + let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to clone repository: {}", e), + ) + })?; + + println!("Repository cloned successfully"); + println!("Repository cloned to: {}", temp_dir.path().display()); + + if branch_name.is_some() || commit_hash.is_some() || tag_name.is_some() { + let mut fetch_opts = FetchOptions::new(); + fetch_opts.remote_callbacks(callbacks); + + let mut remote = repo.find_remote("origin").map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find remote 'origin': {}", e), + ) + })?; + + remote + .fetch( + &["refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"], + Some(&mut fetch_opts), + None, + ) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to fetch all branches and tags: {}", e), + ) + })?; + + if let Some(tag) = tag_name.as_deref() { + let (object, reference) = + repo.revparse_ext(&format!("refs/tags/{}", tag)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find tag '{}': {}", tag, e), + ) + })?; + + repo.checkout_tree(&object, None).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to checkout tag '{}': {}", tag, e), + ) + })?; + + repo.set_head_detached(object.id()).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to tag '{}': {}", tag, e), + ) + })?; + + println!("Checked out tag '{}'", tag); + } else if let Some(commit) = commit_hash.as_deref() { + let oid = Oid::from_str(commit).map_err(|e| { + Error::new( + Span::call_site(), + format!("Invalid commit hash '{}': {}", commit, e), + ) + })?; + + let commit_obj = repo.find_commit(oid).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find commit '{}': {}", commit, e), + ) + })?; + + repo.set_head_detached(commit_obj.id()).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to commit '{}': {}", commit, e), + ) + })?; + + repo.checkout_head(Some(git2::build::CheckoutBuilder::new().force())) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to checkout commit '{}': {}", commit, e), + ) + })?; + + println!("Checked out commit '{}'", commit); + } else if let Some(branch) = branch_name.as_deref() { + let (object, reference) = + repo.revparse_ext(&format!("origin/{}", branch)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to find '{}' branch: {}", branch, e), + ) + })?; + + repo.checkout_tree(&object, None).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to checkout '{}' branch: {}", branch, e), + ) + })?; + + repo.set_head(&format!("refs/heads/{}", branch)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to set HEAD to '{}' branch: {}", branch, e), + ) + })?; + + println!("Switched to '{}' branch", branch); + } + } else { + println!("Cloned default branch only"); + } + + Ok(temp_dir) +} + +pub fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { + let snippets_dir = crate_root.join(".snippets"); + println!("Snippets directory: {}", snippets_dir.display()); + + // Ensure snippets directory exists + fs::create_dir_all(&snippets_dir).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create .snippets directory: {}", e), + ) + })?; + + let snippet_path = generate_snippet_path(&snippets_dir, file_path, item_ident); + println!("Snippet path: {}", snippet_path.display()); + + let has_internet = check_internet_connectivity(); + println!( + "Internet connectivity: {}", + if has_internet { + "Available" + } else { + "Not available" + } + ); + + if !has_internet { + println!("No internet connection, attempting to read from cached snippet"); + // Try to read from the snippet file + match fs::read_to_string(&snippet_path) { + Ok(content) => { + println!("Successfully read content from cached snippet"); + Ok(content) + } + Err(e) => Err(Error::new( + Span::call_site(), + format!( + "No internet connection and failed to read cached snippet at {}: {}. + Please ensure you have internet connectivity for the first run.", + snippet_path.display(), + e + ), + )), + } + } else { + // Internet is available, proceed with full path checking and content updating + let full_path = crate_root.join(file_path); + println!( + "Internet available, checking file at: {}", + full_path.display() + ); + + let existing_content = fs::read_to_string(&snippet_path).ok(); + let new_content = extract_item_from_file(&full_path, item_ident)?; + + if existing_content.as_ref().map(|c| hash_content(c)) != Some(hash_content(&new_content)) { + println!("Updating snippet file with new content"); + fs::write(&snippet_path, &new_content).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to write snippet file: {}", e), + ) + })?; + } else { + println!("Snippet is up to date"); + } + + Ok(new_content) + } +} + +pub fn generate_snippet_path(snippets_dir: &Path, file_path: &str, item_ident: &str) -> PathBuf { + println!( + "inside generate_snippet_path ----> Snippets directory: {}", + snippets_dir.display() + ); + println!( + "inside generate_snippet_path ----> File path: {}", + file_path + ); + println!( + "inside generate_snippet_path ----> Item ident: {}", + item_ident + ); + let path = PathBuf::from(file_path); + let file_name = path + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or("unknown"); + snippets_dir.join(format!(".docify-snippet-{}-{}", file_name, item_ident)) +} +fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { + println!( + "inside extract_item_from_file ----> Extracting item '{}' from '{}'", + item_ident, + file_path.display() + ); + + let source_code = fs::read_to_string(file_path).map_err(|e| { + Error::new( + Span::call_site(), + format!( + "Could not read the specified path '{}': {}", + file_path.display(), + e + ), + ) + })?; + + println!( + "inside extract_item_from_file ----> Source code: {}", + source_code + ); + + let mut visitor = ItemVisitor { + search: syn::parse_str(item_ident)?, + results: Vec::new(), + }; + visitor.visit_file(&syn::parse_file(&source_code)?); + println!( + "inside extract_item_from_file ----> Visitor results: {:?}", + visitor.results + ); + if visitor.results.is_empty() { + return Err(Error::new( + Span::call_site(), + format!( + "Could not find docify export item '{}' in '{}'.", + item_ident, + file_path.display() + ), + )); + } + + println!("Successfully extracted item from file"); + let (item, style) = visitor.results.first().unwrap(); + source_excerpt(&source_code, item, *style) +} + +pub fn hash_content(content: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(content); + let result = format!("{:x}", hasher.finalize()); + println!("Content hash: {}", result); + result +} + +/// Checks if there is an active internet connection by attempting to connect to multiple reliable hosts +pub fn check_internet_connectivity() -> bool { + // List of reliable hosts and ports to try + let hosts = [ + ("8.8.8.8", 53), // Google DNS + ("1.1.1.1", 53), // Cloudflare DNS + ("208.67.222.222", 53), // OpenDNS + ]; + + // Set a timeout for connection attempts + let timeout = Duration::from_secs(1); + + for &(host, port) in hosts.iter() { + if let Ok(stream) = TcpStream::connect((host, port)) { + // Set the timeout for read/write operations + if stream.set_read_timeout(Some(timeout)).is_ok() + && stream.set_write_timeout(Some(timeout)).is_ok() + { + return true; + } + } + } + + false +} From 1106f8e8a6feb6ecf385a942bb91b6829fad889f Mon Sep 17 00:00:00 2001 From: Prakash Date: Thu, 31 Oct 2024 19:05:53 +0530 Subject: [PATCH 23/55] new flow working tested , prettify code and error messages remaining --- macros/Cargo.toml | 2 +- macros/src/lib.rs | 117 +++++++++++++++++++++++---------- macros/src/utils/mod.rs | 140 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 221 insertions(+), 38 deletions(-) diff --git a/macros/Cargo.toml b/macros/Cargo.toml index eb61498..c6406db 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -23,5 +23,5 @@ termcolor = "1" once_cell = "1" toml = "0.8" tempfile = "3.3.0" -git2 = "0.16.0" +git2 = "0.18" sha2 = "0.10.6" diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 126b74e..dfa4162 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1,16 +1,12 @@ mod utils; use common_path::common_path; use derive_syn_parse::Parse; -use git2::{FetchOptions, ObjectType, Oid, RemoteCallbacks, Repository}; use once_cell::sync::Lazy; use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, ToTokens}; use regex::Regex; -use sha2::{Digest, Sha256}; -use std::net::TcpStream; -use std::sync::Mutex; -use std::time::Duration; + use std::{ cmp::min, collections::HashMap, @@ -26,10 +22,8 @@ use syn::{ spanned::Spanned, token::Paren, visit::{self, Visit}, - AttrStyle, Attribute, Error, File, Ident, ImplItem, Item, LitStr, Meta, Result, Token, - TraitItem, + AttrStyle, Attribute, Error, Ident, ImplItem, Item, LitStr, Meta, Result, Token, TraitItem, }; -use tempfile::{Builder, TempDir}; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use toml::{Table, Value}; use utils::*; @@ -1129,35 +1123,90 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } println!("Detected git-based embedding"); - println!("Detected git-based embedding"); - let repo_path = get_or_clone_repo( - git_url.value().as_str(), - &crate_root, - args.branch_name.as_ref().map(|b| b.value()), - args.commit_hash.as_ref().map(|c| c.value()), - args.tag_name.as_ref().map(|t| t.value()), - )?; - - let file_path = args.file_path.value().to_string(); - let full_path = repo_path.join(&file_path).to_string_lossy().into_owned(); - println!("embed_internal_str ----> Full path: {}", full_path); - let snippet_content = match &args.item_ident { - Some(ident) => { - println!("Generating snippet for item: {}", ident); - manage_snippet(&crate_root, &full_path, &ident.to_string())? - } - None => { - println!("No specific item requested, using entire file content"); - fs::read_to_string(repo_path.join(&file_path)).map_err(|e| { - Error::new(Span::call_site(), format!("Failed to read file: {}", e)) - })? - } + println!("Starting git-based embedding process..."); + // Get commit SHA without cloning + println!("Fetching commit SHA without cloning repository..."); + // Get commit SHA - either directly from args or by fetching + let commit_sha = if let Some(hash) = args.commit_hash.as_ref() { + println!("Using provided commit hash"); + hash.value().to_string() + } else { + println!("Fetching commit SHA without cloning repository..."); + get_remote_commit_sha_without_clone( + git_url.value().as_str(), + args.branch_name + .as_ref() + .map(|b| b.value().to_string()) + .as_deref(), + args.tag_name + .as_ref() + .map(|t| t.value().to_string()) + .as_deref(), + )? }; + println!("Retrieved commit SHA: {}", commit_sha); + + // Generate deterministic filename + println!("Generating deterministic filename for snippet..."); + let snippet_filename = generate_snippet_filename(&commit_sha, &args.file_path.value()); + println!("Generated snippet filename: {}", snippet_filename); + + let snippets_dir = caller_crate_root() + .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))? + .join(".snippets"); + println!("Using snippets directory: {}", snippets_dir.display()); - let formatted = fix_indentation(&snippet_content); - let output = into_example(&formatted, lang); + fs::create_dir_all(&snippets_dir).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create snippets directory: {}", e), + ) + })?; + println!("Created snippets directory if it didn't exist"); + + let snippet_path = snippets_dir.join(&snippet_filename); + println!("Full snippet path: {}", snippet_path.display()); + + if !snippet_path.exists() { + println!("Snippet file does not exist, creating new one..."); + // Clone repo and checkout specific commit + println!("Cloning repository and checking out commit..."); + let temp_dir = clone_and_checkout_repo(git_url.value().as_str(), &commit_sha)?; + println!( + "Repository cloned to temporary directory: {}", + temp_dir.path().display() + ); + + // Copy file to snippets directory + let source_path = temp_dir.path().join(&args.file_path.value()); + println!("Reading source file from: {}", source_path.display()); + let content = fs::read_to_string(&source_path).map_err(|e| { + Error::new( + args.file_path.span(), + format!("Failed to read file from repo: {}", e), + ) + })?; + println!("Successfully read source file content"); + + println!("Writing content to snippet file..."); + fs::write(&snippet_path, content).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to write snippet file: {}", e), + ) + })?; + println!("Successfully wrote snippet file"); + } else { + println!("Using existing snippet file"); + } + + let file_content_with_ident = extract_item_from_file( + &snippet_path, + &args.item_ident.as_ref().unwrap().to_string(), + )?; - println!("Successfully embedded git-based content"); + let formatted_content = fix_indentation(&file_content_with_ident); + let output = into_example(&formatted_content, lang); println!( "embed_internal_str ----> Final output length: {}", output.len() diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index a1b767b..c2acaef 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -127,7 +127,7 @@ pub fn clone_repo( })?; if let Some(tag) = tag_name.as_deref() { - let (object, reference) = + let (object, _reference) = repo.revparse_ext(&format!("refs/tags/{}", tag)) .map_err(|e| { Error::new( @@ -183,7 +183,7 @@ pub fn clone_repo( println!("Checked out commit '{}'", commit); } else if let Some(branch) = branch_name.as_deref() { - let (object, reference) = + let (object, _reference) = repo.revparse_ext(&format!("origin/{}", branch)) .map_err(|e| { Error::new( @@ -306,7 +306,7 @@ pub fn generate_snippet_path(snippets_dir: &Path, file_path: &str, item_ident: & .unwrap_or("unknown"); snippets_dir.join(format!(".docify-snippet-{}-{}", file_name, item_ident)) } -fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { +pub fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { println!( "inside extract_item_from_file ----> Extracting item '{}' from '{}'", item_ident, @@ -387,3 +387,137 @@ pub fn check_internet_connectivity() -> bool { false } + +/// Helper function to convert git2::Error to syn::Error +fn git_err_to_syn(err: git2::Error) -> syn::Error { + syn::Error::new(Span::call_site(), format!("Git error: {}", err)) +} + +/// Helper function to convert std::io::Error to syn::Error +fn io_err_to_syn(err: std::io::Error) -> syn::Error { + syn::Error::new(Span::call_site(), format!("IO error: {}", err)) +} + +/// Gets commit SHA without cloning entire repo +pub fn get_remote_commit_sha_without_clone( + git_url: &str, + branch: Option<&str>, + tag: Option<&str>, +) -> Result { + let temp_dir = tempfile::Builder::new() + .prefix("docify-temp-") + .rand_bytes(5) + .tempdir() + .map_err(|e| { + syn::Error::new( + Span::call_site(), + format!("Failed to create temp dir: {}", e), + ) + })?; + + let repo = Repository::init(temp_dir.path()).map_err(git_err_to_syn)?; + let mut remote = repo.remote_anonymous(git_url).map_err(git_err_to_syn)?; + + let refspecs = if let Some(tag_name) = tag { + vec![format!("refs/tags/{}:refs/tags/{}", tag_name, tag_name)] + } else { + let branch_name = branch.unwrap_or("HEAD"); + vec![format!( + "refs/heads/{}:refs/heads/{}", + branch_name, branch_name + )] + }; + + remote + .fetch( + refspecs + .iter() + .map(|s| s.as_str()) + .collect::>() + .as_slice(), + None, + None, + ) + .map_err(git_err_to_syn)?; + + let commit_id = if let Some(tag_name) = tag { + let tag_ref = repo + .find_reference(&format!("refs/tags/{}", tag_name)) + .map_err(git_err_to_syn)?; + tag_ref.peel_to_commit().map_err(git_err_to_syn)?.id() + } else { + let branch_name = branch.unwrap_or("HEAD"); + let reference = repo + .find_reference(&format!("refs/heads/{}", branch_name)) + .map_err(git_err_to_syn)?; + reference.peel_to_commit().map_err(git_err_to_syn)?.id() + }; + + Ok(commit_id.to_string()) +} + +/// Clones repo and checks out specific commit +pub fn clone_and_checkout_repo(git_url: &str, commit_sha: &str) -> Result { + let temp_dir = tempfile::Builder::new() + .prefix("docify-temp-") + .rand_bytes(5) + .tempdir() + .map_err(|e| { + syn::Error::new( + Span::call_site(), + format!("Failed to create temp dir: {}", e), + ) + })?; + + let repo = Repository::init(temp_dir.path()).map_err(git_err_to_syn)?; + + let mut callbacks = RemoteCallbacks::new(); + callbacks.transfer_progress(|p| { + println!( + "Fetching: {}/{} objects", + p.received_objects(), + p.total_objects() + ); + true + }); + + let mut fetch_opts = FetchOptions::new(); + fetch_opts.remote_callbacks(callbacks); + fetch_opts.depth(1); + + let mut remote = repo.remote_anonymous(git_url).map_err(git_err_to_syn)?; + + remote + .fetch( + &[&format!("+{commit_sha}:refs/heads/temp")], + Some(&mut fetch_opts), + None, + ) + .map_err(git_err_to_syn)?; + + let commit_id = git2::Oid::from_str(commit_sha).map_err(git_err_to_syn)?; + let commit = repo.find_commit(commit_id).map_err(git_err_to_syn)?; + let tree = commit.tree().map_err(git_err_to_syn)?; + repo.checkout_tree(tree.as_object(), None) + .map_err(git_err_to_syn)?; + repo.set_head_detached(commit_id).map_err(git_err_to_syn)?; + + Ok(temp_dir) +} + +/// Generates a deterministic filename for the snippet +pub fn generate_snippet_filename(commit_sha: &str, path: &str) -> String { + let path_buf = PathBuf::from(path); + let file_name = path_buf + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or("unknown"); + + format!("{}-{}-{}", &commit_sha[..8], hash_path(path), file_name) +} + +fn hash_path(path: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(path.as_bytes()); + format!("{:.8x}", hasher.finalize()) // First 8 chars of hash +} From ed70fa0f85093314de679461bc66663047742083 Mon Sep 17 00:00:00 2001 From: Prakash Date: Thu, 31 Oct 2024 21:18:24 +0530 Subject: [PATCH 24/55] git dir cache --- macros/src/lib.rs | 13 +- macros/src/utils/mod.rs | 268 +++++++--------------------------------- 2 files changed, 52 insertions(+), 229 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index dfa4162..3c80441 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1169,16 +1169,13 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if !snippet_path.exists() { println!("Snippet file does not exist, creating new one..."); - // Clone repo and checkout specific commit - println!("Cloning repository and checking out commit..."); - let temp_dir = clone_and_checkout_repo(git_url.value().as_str(), &commit_sha)?; - println!( - "Repository cloned to temporary directory: {}", - temp_dir.path().display() - ); + // Clone repo and checkout specific commit, reusing existing clone if available + println!("Cloning/reusing repository and checking out commit..."); + let repo_dir = clone_and_checkout_repo(git_url.value().as_str(), &commit_sha)?; + println!("Using repository at directory: {}", repo_dir.display()); // Copy file to snippets directory - let source_path = temp_dir.path().join(&args.file_path.value()); + let source_path = repo_dir.join(&args.file_path.value()); println!("Reading source file from: {}", source_path.display()); let content = fs::read_to_string(&source_path).map_err(|e| { Error::new( diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index c2acaef..24fc2eb 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -1,221 +1,16 @@ use git2::{FetchOptions, Oid, RemoteCallbacks, Repository}; -use once_cell::sync::Lazy; use proc_macro2::Span; use sha2::{Digest, Sha256}; -use std::collections::HashMap; use std::fs; use std::net::TcpStream; use std::path::{Path, PathBuf}; -use std::sync::Mutex; use std::time::Duration; use syn::visit::Visit; use syn::Error; use syn::Result; -use tempfile::{Builder, TempDir}; use crate::{source_excerpt, ItemVisitor}; -// Cache for storing repository clones -static REPO_CACHE: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); - -// Generate a unique cache key for a repository -pub fn generate_cache_key( - git_url: &str, - branch: Option<&str>, - commit: Option<&str>, - tag: Option<&str>, -) -> String { - format!( - "{}:{}:{}:{}", - git_url, - branch.unwrap_or("master"), - commit.unwrap_or("none"), - tag.unwrap_or("none") - ) -} -pub fn get_or_clone_repo( - git_url: &str, - project_root: &PathBuf, - branch_name: Option, - commit_hash: Option, - tag_name: Option, -) -> Result { - let cache_key = generate_cache_key( - git_url, - branch_name.as_deref(), - commit_hash.as_deref(), - tag_name.as_deref(), - ); - - let mut cache = REPO_CACHE.lock().unwrap(); - - if !cache.contains_key(&cache_key) { - println!("Cache miss for {}, cloning repository...", git_url); - let temp_dir = clone_repo(git_url, project_root, branch_name, commit_hash, tag_name)?; - cache.insert(cache_key.clone(), temp_dir); - } else { - println!("Cache hit for {}, using existing clone", git_url); - } - - Ok(cache.get(&cache_key).unwrap().path().to_path_buf()) -} -pub fn clone_repo( - git_url: &str, - _project_root: &PathBuf, - branch_name: Option, - commit_hash: Option, - tag_name: Option, -) -> Result { - let temp_dir = Builder::new() - .prefix("docify-") - .rand_bytes(5) - .tempdir() - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create temp directory: {}", e), - ) - })?; - - println!( - "Temporary directory created at: {}", - temp_dir.path().display() - ); - - let mut callbacks = RemoteCallbacks::new(); - callbacks.transfer_progress(|progress| { - println!( - "Transfer progress: {}/{} objects", - progress.received_objects(), - progress.total_objects() - ); - true - }); - - let repo = Repository::clone(git_url, temp_dir.path()).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to clone repository: {}", e), - ) - })?; - - println!("Repository cloned successfully"); - println!("Repository cloned to: {}", temp_dir.path().display()); - - if branch_name.is_some() || commit_hash.is_some() || tag_name.is_some() { - let mut fetch_opts = FetchOptions::new(); - fetch_opts.remote_callbacks(callbacks); - - let mut remote = repo.find_remote("origin").map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find remote 'origin': {}", e), - ) - })?; - - remote - .fetch( - &["refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"], - Some(&mut fetch_opts), - None, - ) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to fetch all branches and tags: {}", e), - ) - })?; - - if let Some(tag) = tag_name.as_deref() { - let (object, _reference) = - repo.revparse_ext(&format!("refs/tags/{}", tag)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find tag '{}': {}", tag, e), - ) - })?; - - repo.checkout_tree(&object, None).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout tag '{}': {}", tag, e), - ) - })?; - - repo.set_head_detached(object.id()).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to set HEAD to tag '{}': {}", tag, e), - ) - })?; - - println!("Checked out tag '{}'", tag); - } else if let Some(commit) = commit_hash.as_deref() { - let oid = Oid::from_str(commit).map_err(|e| { - Error::new( - Span::call_site(), - format!("Invalid commit hash '{}': {}", commit, e), - ) - })?; - - let commit_obj = repo.find_commit(oid).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find commit '{}': {}", commit, e), - ) - })?; - - repo.set_head_detached(commit_obj.id()).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to set HEAD to commit '{}': {}", commit, e), - ) - })?; - - repo.checkout_head(Some(git2::build::CheckoutBuilder::new().force())) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout commit '{}': {}", commit, e), - ) - })?; - - println!("Checked out commit '{}'", commit); - } else if let Some(branch) = branch_name.as_deref() { - let (object, _reference) = - repo.revparse_ext(&format!("origin/{}", branch)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to find '{}' branch: {}", branch, e), - ) - })?; - - repo.checkout_tree(&object, None).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to checkout '{}' branch: {}", branch, e), - ) - })?; - - repo.set_head(&format!("refs/heads/{}", branch)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to set HEAD to '{}' branch: {}", branch, e), - ) - })?; - - println!("Switched to '{}' branch", branch); - } - } else { - println!("Cloned default branch only"); - } - - Ok(temp_dir) -} - pub fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { let snippets_dir = crate_root.join(".snippets"); println!("Snippets directory: {}", snippets_dir.display()); @@ -393,11 +188,6 @@ fn git_err_to_syn(err: git2::Error) -> syn::Error { syn::Error::new(Span::call_site(), format!("Git error: {}", err)) } -/// Helper function to convert std::io::Error to syn::Error -fn io_err_to_syn(err: std::io::Error) -> syn::Error { - syn::Error::new(Span::call_site(), format!("IO error: {}", err)) -} - /// Gets commit SHA without cloning entire repo pub fn get_remote_commit_sha_without_clone( git_url: &str, @@ -414,10 +204,13 @@ pub fn get_remote_commit_sha_without_clone( format!("Failed to create temp dir: {}", e), ) })?; + println!("Created temp dir: {}", temp_dir.path().display()); let repo = Repository::init(temp_dir.path()).map_err(git_err_to_syn)?; let mut remote = repo.remote_anonymous(git_url).map_err(git_err_to_syn)?; + println!("Initialized repo"); + let refspecs = if let Some(tag_name) = tag { vec![format!("refs/tags/{}:refs/tags/{}", tag_name, tag_name)] } else { @@ -428,6 +221,7 @@ pub fn get_remote_commit_sha_without_clone( )] }; + println!("Fetching refs"); remote .fetch( refspecs @@ -440,6 +234,7 @@ pub fn get_remote_commit_sha_without_clone( ) .map_err(git_err_to_syn)?; + println!("Fetching commit SHA"); let commit_id = if let Some(tag_name) = tag { let tag_ref = repo .find_reference(&format!("refs/tags/{}", tag_name)) @@ -456,20 +251,49 @@ pub fn get_remote_commit_sha_without_clone( Ok(commit_id.to_string()) } -/// Clones repo and checks out specific commit -pub fn clone_and_checkout_repo(git_url: &str, commit_sha: &str) -> Result { - let temp_dir = tempfile::Builder::new() - .prefix("docify-temp-") - .rand_bytes(5) - .tempdir() - .map_err(|e| { - syn::Error::new( +pub fn get_or_create_commit_dir(git_url: &str, commit_sha: &str) -> Result { + let temp_base = std::env::temp_dir().join("docify-repos"); + + // Extract repo name from git URL + let repo_name = git_url + .split('/') + .last() + .and_then(|s| s.strip_suffix(".git")) + .unwrap_or("repo"); + + // Use first 8 chars of commit hash + let short_commit = &commit_sha[..8]; + + // Create directory name: docify-{short_commit}-{repo_name} + let dir_name = format!("docify-{}-{}", short_commit, repo_name); + let commit_dir = temp_base.join(dir_name); + + if commit_dir.exists() { + println!("Found existing repo directory: {}", commit_dir.display()); + Ok(commit_dir) + } else { + println!("Creating new repo directory: {}", commit_dir.display()); + fs::create_dir_all(&commit_dir).map_err(|e| { + Error::new( Span::call_site(), - format!("Failed to create temp dir: {}", e), + format!("Failed to create commit directory: {}", e), ) })?; + Ok(commit_dir) + } +} - let repo = Repository::init(temp_dir.path()).map_err(git_err_to_syn)?; +/// Clones repo and checks out specific commit, reusing existing clone if available +pub fn clone_and_checkout_repo(git_url: &str, commit_sha: &str) -> Result { + let commit_dir = get_or_create_commit_dir(git_url, commit_sha)?; + + // Check if repo is already cloned and checked out + if commit_dir.join(".git").exists() { + println!("Using existing repo clone at: {}", commit_sha); + return Ok(commit_dir); + } + + println!("Cloning new repo for commit: {}", commit_sha); let mut callbacks = RemoteCallbacks::new(); callbacks.transfer_progress(|p| { @@ -485,6 +309,7 @@ pub fn clone_and_checkout_repo(git_url: &str, commit_sha: &str) -> Result Result Date: Thu, 31 Oct 2024 22:56:07 +0530 Subject: [PATCH 25/55] default branch working tested --- macros/src/utils/mod.rs | 55 ++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 24fc2eb..a934d88 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -1,4 +1,4 @@ -use git2::{FetchOptions, Oid, RemoteCallbacks, Repository}; +use git2::{Direction, FetchOptions, Oid, RemoteCallbacks, Repository}; use proc_macro2::Span; use sha2::{Digest, Sha256}; use std::fs; @@ -182,7 +182,7 @@ pub fn check_internet_connectivity() -> bool { false } - +// comment /// Helper function to convert git2::Error to syn::Error fn git_err_to_syn(err: git2::Error) -> syn::Error { syn::Error::new(Span::call_site(), format!("Git error: {}", err)) @@ -199,7 +199,7 @@ pub fn get_remote_commit_sha_without_clone( .rand_bytes(5) .tempdir() .map_err(|e| { - syn::Error::new( + Error::new( Span::call_site(), format!("Failed to create temp dir: {}", e), ) @@ -211,17 +211,40 @@ pub fn get_remote_commit_sha_without_clone( println!("Initialized repo"); + // First, fetch the remote HEAD to determine default branch + println!("ā„¹ļø Fetching remote references..."); + remote.connect(Direction::Fetch).map_err(git_err_to_syn)?; + + // Handle default branch resolution with proper error conversion + let default_branch = remote + .default_branch() + .map_err(git_err_to_syn)? + .as_str() + .map(String::from) + .ok_or_else(|| Error::new(Span::call_site(), "Invalid default branch name"))?; + + print!("defult branch --------> {}", default_branch); + remote.disconnect().map_err(git_err_to_syn)?; + + // Convert refs/heads/main to just main + let default_branch = default_branch + .strip_prefix("refs/heads/") + .unwrap_or(&default_branch); + + println!("ā„¹ļø Default branch: {}", default_branch); + + // Determine which refs to fetch let refspecs = if let Some(tag_name) = tag { vec![format!("refs/tags/{}:refs/tags/{}", tag_name, tag_name)] } else { - let branch_name = branch.unwrap_or("HEAD"); + let branch_name = branch.unwrap_or(default_branch); vec![format!( "refs/heads/{}:refs/heads/{}", branch_name, branch_name )] }; - println!("Fetching refs"); + println!("Fetching refs: {:?}", refspecs); remote .fetch( refspecs @@ -234,14 +257,14 @@ pub fn get_remote_commit_sha_without_clone( ) .map_err(git_err_to_syn)?; - println!("Fetching commit SHA"); + // Determine which commit to use let commit_id = if let Some(tag_name) = tag { let tag_ref = repo .find_reference(&format!("refs/tags/{}", tag_name)) .map_err(git_err_to_syn)?; tag_ref.peel_to_commit().map_err(git_err_to_syn)?.id() } else { - let branch_name = branch.unwrap_or("HEAD"); + let branch_name = branch.unwrap_or(default_branch); let reference = repo .find_reference(&format!("refs/heads/{}", branch_name)) .map_err(git_err_to_syn)?; @@ -258,8 +281,18 @@ pub fn get_or_create_commit_dir(git_url: &str, commit_sha: &str) -> Result {}", + repo_name + ); // Use first 8 chars of commit hash let short_commit = &commit_sha[..8]; @@ -289,7 +322,11 @@ pub fn clone_and_checkout_repo(git_url: &str, commit_sha: &str) -> Result: {} returning existing dir path ----> {}", + commit_sha, + commit_dir.display() + ); return Ok(commit_dir); } From 1ffa78a77d82136e180664aa2933d182b6ace14f Mon Sep 17 00:00:00 2001 From: Prakash Date: Fri, 1 Nov 2024 02:39:57 +0530 Subject: [PATCH 26/55] removed oid from imports --- macros/src/utils/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index a934d88..acdcd57 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -1,4 +1,4 @@ -use git2::{Direction, FetchOptions, Oid, RemoteCallbacks, Repository}; +use git2::{Direction, FetchOptions, RemoteCallbacks, Repository}; use proc_macro2::Span; use sha2::{Digest, Sha256}; use std::fs; From d8b425c5eaef90824735f634393fc2ad32d57e8d Mon Sep 17 00:00:00 2001 From: Prakash Date: Fri, 1 Nov 2024 23:28:14 +0530 Subject: [PATCH 27/55] minimal flow working , caching case remaining where if the snippet is found don't clone --- macros/src/lib.rs | 229 ++++++++++++++++++++++++++-------------- macros/src/utils/mod.rs | 182 +++++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+), 77 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 3c80441..d68e95d 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1101,56 +1101,130 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(git_url) = &args.git_url { let has_internet = check_internet_connectivity(); + // Determine git option type and value + println!( + "\nšŸŒ Internet connectivity: {}", + if has_internet { "Online" } else { "Offline" } + ); + // Determine git option type and value + let (git_option_type, git_option_value) = if let Some(hash) = &args.commit_hash { + println!("Using commit hash: {}", hash.value()); + ("commit".to_string(), hash.value()) + } else if let Some(tag) = &args.tag_name { + println!("Using tag: {}", tag.value()); + ("tag".to_string(), tag.value()) + } else { + // Branch case: Use provided branch or fallback + let branch_name = if let Some(ref branch) = args.branch_name { + println!("Using provided branch: {}", branch.value()); + branch.value() + } else if has_internet { + // No branch provided and online: try to get default branch + match get_default_branch(git_url.value().as_str()) { + Ok(branch) => { + println!("Using default branch: {}", branch); + branch + } + Err(_) => { + println!("Failed to get default branch, falling back to master"); + "master".to_string() + } + } + } else { + println!("Offline mode, using master as fallback"); + "master".to_string() + }; + ("branch".to_string(), branch_name) + }; - if !has_internet { - if !has_internet { - println!("No internet connection detected. Using cached snippet if available in the snippets dir."); - // Get content from snippet and process it - let content = manage_snippet( - &crate_root, - &args.file_path.value(), - args.item_ident - .as_ref() - .map(|i| i.to_string()) - .unwrap_or_default() - .as_str(), - )?; - - // Fix indentation and process the content - let fixed = fix_indentation(&content); - return Ok(into_example(&fixed, lang)); - } - } + println!( + "Git option type: {}, value: {}", + git_option_type, git_option_value + ); - println!("Detected git-based embedding"); - println!("Starting git-based embedding process..."); - // Get commit SHA without cloning - println!("Fetching commit SHA without cloning repository..."); - // Get commit SHA - either directly from args or by fetching - let commit_sha = if let Some(hash) = args.commit_hash.as_ref() { - println!("Using provided commit hash"); - hash.value().to_string() + // need to add a check here if internet is not present but commit hash is provided + // Create snippet file object based on connectivity + let new_snippet = if has_internet { + println!("\nšŸ” Fetching latest commit SHA from remote repository..."); + let commit_sha = if let Some(hash) = &args.commit_hash { + hash.value() + } else if let Some(tag) = &args.tag_name { + get_remote_commit_sha_without_clone( + git_url.value().as_str(), + None, + Some(tag.value().as_str()), + )? + } else { + get_remote_commit_sha_without_clone( + git_url.value().as_str(), + Some(git_option_value.as_str()), + None, + )? + }; + println!("āœ… Found commit SHA: {}", commit_sha); + + SnippetFile::new_with_commit( + git_url.value().as_str(), + &git_option_type, + &git_option_value, + &args.file_path.value(), + &commit_sha, + ) } else { - println!("Fetching commit SHA without cloning repository..."); - get_remote_commit_sha_without_clone( + SnippetFile::new_without_commit( git_url.value().as_str(), - args.branch_name - .as_ref() - .map(|b| b.value().to_string()) - .as_deref(), - args.tag_name - .as_ref() - .map(|t| t.value().to_string()) - .as_deref(), - )? + &git_option_type, + &git_option_value, + &args.file_path.value(), + ) }; - println!("Retrieved commit SHA: {}", commit_sha); - // Generate deterministic filename - println!("Generating deterministic filename for snippet..."); - let snippet_filename = generate_snippet_filename(&commit_sha, &args.file_path.value()); - println!("Generated snippet filename: {}", snippet_filename); + println!("\nšŸ” Checking for existing snippets..."); + // Check for existing snippet with same prefix + if let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) { + if !has_internet { + println!( + "āœ… Found existing snippet (offline mode): .snippets/{}", + existing_snippet.full_name + ); + } + + // Online mode comparison + if let (Some(existing_hash), Some(new_hash)) = + (&existing_snippet.commit_hash, &new_snippet.commit_hash) + { + if existing_hash == new_hash { + println!( + "āœ… Existing snippet is up to date at just return it and read ident from it no need to clone: .snippets/{}", + existing_snippet.full_name + ); + } else { + println!("ā„¹ļø Found existing snippet with different commit hash:"); + println!(" Current: {}", existing_hash); + println!(" New: {}", new_hash); + println!("šŸ”„ Updating snippet..."); + + fs::remove_file(Path::new(".snippets").join(&existing_snippet.full_name)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to remove old snippet file: {}", e), + ) + })?; + println!("āœ… Removed old snippet file"); + println!("here you would clone again and update the file") + } + } + } + if !has_internet { + return Err(Error::new( + Span::call_site(), + "No matching snippet found and no internet connection available", + )); + } + + // creating snippets dir if it doesn't exist let snippets_dir = caller_crate_root() .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))? .join(".snippets"); @@ -1162,40 +1236,41 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - format!("Failed to create snippets directory: {}", e), ) })?; - println!("Created snippets directory if it didn't exist"); - - let snippet_path = snippets_dir.join(&snippet_filename); - println!("Full snippet path: {}", snippet_path.display()); - - if !snippet_path.exists() { - println!("Snippet file does not exist, creating new one..."); - // Clone repo and checkout specific commit, reusing existing clone if available - println!("Cloning/reusing repository and checking out commit..."); - let repo_dir = clone_and_checkout_repo(git_url.value().as_str(), &commit_sha)?; - println!("Using repository at directory: {}", repo_dir.display()); - - // Copy file to snippets directory - let source_path = repo_dir.join(&args.file_path.value()); - println!("Reading source file from: {}", source_path.display()); - let content = fs::read_to_string(&source_path).map_err(|e| { - Error::new( - args.file_path.span(), - format!("Failed to read file from repo: {}", e), - ) - })?; - println!("Successfully read source file content"); - - println!("Writing content to snippet file..."); - fs::write(&snippet_path, content).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to write snippet file: {}", e), - ) - })?; - println!("Successfully wrote snippet file"); - } else { - println!("Using existing snippet file"); - } + + println!("cloning and checking out repo..."); + + // returns the directory of the cloned repo with caching + let repo_dir = clone_and_checkout_repo( + git_url.value().as_str(), + &new_snippet.commit_hash.as_ref().unwrap(), + )?; + println!("āœ… Cloned and checked out repo"); + + let source_path = repo_dir.join(&args.file_path.value()); + println!("Reading source file from: {}", source_path.display()); + + let content = fs::read_to_string(&source_path).map_err(|e| { + Error::new( + args.file_path.span(), + format!("Failed to read file from repo: {}", e), + ) + })?; + + let snippet_path = snippets_dir.join(&new_snippet.full_name); + println!( + "Writing content to snippet file: {}", + snippet_path.display() + ); + fs::write(&snippet_path, content).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to write snippet file: {}", e), + ) + })?; + println!( + "āœ… Wrote content to snippet file at path: {}", + snippet_path.display() + ); let file_content_with_ident = extract_item_from_file( &snippet_path, diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index acdcd57..36add42 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -384,3 +384,185 @@ fn hash_path(path: &str) -> String { hasher.update(path.as_bytes()); format!("{:.8x}", hasher.finalize()) // First 8 chars of hash } + +/// Represents a parsed snippet filename +pub struct SnippetFile { + pub prefix: String, + pub commit_hash: Option, + pub full_name: String, +} + +/// Functions to handle snippet file operations +impl SnippetFile { + pub fn new_without_commit( + git_url: &str, + git_option_type: &str, + git_option_value: &str, + path: &str, + ) -> Self { + println!("\nšŸ“ Creating new SnippetFile.. name without commit."); + println!("ā„¹ļø Input path: {}", path); + + let path_buf = PathBuf::from(path); + let file_name = path_buf + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or("unknown"); + println!("ā„¹ļø Extracted filename: {}", file_name); + + let prefix = format!( + "{}-{}-{}-{}", + hash_git_url(git_url), + hash_git_option(git_option_type, git_option_value), + hash_string(path), + file_name, + ); + println!("ā„¹ļø Generated prefix: {}", prefix); + + Self { + prefix: prefix.clone(), + commit_hash: None, + full_name: prefix, + } + } + + pub fn new_with_commit( + git_url: &str, + git_option_type: &str, + git_option_value: &str, + path: &str, + commit_sha: &str, + ) -> Self { + println!("\nšŸ“ Creating new SnippetFile.. name with commit."); + let base = Self::new_without_commit(git_url, git_option_type, git_option_value, path); + let full_name = format!("{}-{}.rs", base.prefix, commit_sha); + println!("āœ… Created snippet filename: {}", full_name); + + Self { + prefix: base.prefix, + commit_hash: Some(commit_sha.to_string()), + full_name, + } + } + + pub fn find_existing(prefix: &str) -> Option { + println!("\nšŸ” Looking for existing snippet with prefix: {}", prefix); + + // Get the crate root path + let crate_root = match crate::caller_crate_root() { + Some(root) => root, + None => { + println!("āŒ Failed to resolve crate root"); + return None; + } + }; + + // Use absolute path by joining with crate root + let snippets_dir = crate_root.join(".snippets"); + println!("šŸ“ Checking snippets directory: {}", snippets_dir.display()); + + // Check if directory exists and is actually a directory + if !snippets_dir.exists() { + println!( + "āŒ .snippets directory does not exist at {}", + snippets_dir.display() + ); + return None; + } + + fs::read_dir(snippets_dir).ok()?.find_map(|entry| { + let entry = entry.ok()?; + println!("entry: {:?}", entry); + let file_name = entry.file_name().to_string_lossy().to_string(); + + println!("ā„¹ļø Checking file: {}", file_name); + + if file_name.starts_with(prefix) { + println!("āœ… Found matching file!"); + // Extract commit hash from filename if it exists + let commit_hash = file_name + .strip_suffix(".rs")? + .rsplit('-') + .next() + .map(|s| s.to_string()); + println!( + "ā„¹ļø Extracted commit hash from existing file: {:?}", + commit_hash + ); + + Some(Self { + prefix: prefix.to_string(), + commit_hash, + full_name: file_name, + }) + } else { + None + } + }) + } +} + +fn hash_string(input: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(input.as_bytes()); + format!("{:.8x}", hasher.finalize()) +} + +fn hash_git_option(option_type: &str, value: &str) -> String { + hash_string(&format!("{}-{}", option_type, value)) +} + +/// Helper function to get default branch + +pub fn get_default_branch(git_url: &str) -> Result { + let temp_dir = tempfile::Builder::new() + .prefix("docify-temp-") + .rand_bytes(5) + .tempdir() + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create temp dir: {}", e), + ) + })?; + + let repo = Repository::init(temp_dir.path()) + .map_err(|e| Error::new(Span::call_site(), format!("Failed to init repo: {}", e)))?; + + let mut remote = repo + .remote_anonymous(git_url) + .map_err(|e| Error::new(Span::call_site(), format!("Failed to create remote: {}", e)))?; + + remote.connect(git2::Direction::Fetch).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to connect to remote: {}", e), + ) + })?; + + let default_branch = remote + .default_branch() + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to get default branch: {}", e), + ) + })? + .as_str() + .ok_or_else(|| Error::new(Span::call_site(), "Invalid default branch name"))? + .to_string(); + + remote + .disconnect() + .map_err(|e| Error::new(Span::call_site(), format!("Failed to disconnect: {}", e)))?; + + Ok(default_branch + .strip_prefix("refs/heads/") + .unwrap_or(&default_branch) + .to_string()) +} + +fn hash_git_url(url: &str) -> String { + println!("ā„¹ļø Hashing git URL: {}", url); + hash_string(url) +} From 480aa922e3aa48d97db8291c38ea78805fa2eaed Mon Sep 17 00:00:00 2001 From: Prakash Date: Fri, 1 Nov 2024 23:56:29 +0530 Subject: [PATCH 28/55] caching working edge cases testing remaining --- macros/src/lib.rs | 113 ++++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index d68e95d..cafe1bf 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1181,49 +1181,6 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - println!("\nšŸ” Checking for existing snippets..."); - // Check for existing snippet with same prefix - if let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) { - if !has_internet { - println!( - "āœ… Found existing snippet (offline mode): .snippets/{}", - existing_snippet.full_name - ); - } - - // Online mode comparison - if let (Some(existing_hash), Some(new_hash)) = - (&existing_snippet.commit_hash, &new_snippet.commit_hash) - { - if existing_hash == new_hash { - println!( - "āœ… Existing snippet is up to date at just return it and read ident from it no need to clone: .snippets/{}", - existing_snippet.full_name - ); - } else { - println!("ā„¹ļø Found existing snippet with different commit hash:"); - println!(" Current: {}", existing_hash); - println!(" New: {}", new_hash); - println!("šŸ”„ Updating snippet..."); - - fs::remove_file(Path::new(".snippets").join(&existing_snippet.full_name)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to remove old snippet file: {}", e), - ) - })?; - println!("āœ… Removed old snippet file"); - println!("here you would clone again and update the file") - } - } - } - if !has_internet { - return Err(Error::new( - Span::call_site(), - "No matching snippet found and no internet connection available", - )); - } - // creating snippets dir if it doesn't exist let snippets_dir = caller_crate_root() .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))? @@ -1237,9 +1194,77 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ) })?; - println!("cloning and checking out repo..."); + // Check for existing snippet + let existing_snippet_path = + if let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) { + if !has_internet { + println!( + "āœ… Found existing snippet (offline mode): .snippets/{}", + existing_snippet.full_name + ); + Some(existing_snippet.full_name) + } else { + // Online mode comparison + if let (Some(existing_hash), Some(new_hash)) = + (&existing_snippet.commit_hash, &new_snippet.commit_hash) + { + if existing_hash == new_hash { + println!( + "āœ… Existing snippet is up to date at: .snippets/{}", + existing_snippet.full_name + ); + Some(existing_snippet.full_name) + } else { + println!("ā„¹ļø Found existing snippet with different commit hash:"); + println!(" Current: {}", existing_hash); + println!(" New: {}", new_hash); + println!("šŸ”„ Removing outdated snippet..."); + + // Remove old snippet file + fs::remove_file(snippets_dir.join(&existing_snippet.full_name)) + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to remove old snippet file: {}", e), + ) + })?; + None + } + } else { + None + } + } + } else if !has_internet { + return Err(Error::new( + Span::call_site(), + "No matching snippet found and no internet connection available", + )); + } else { + None + }; + + println!("existing_snippet_path: ----> {:?}", existing_snippet_path); + + // Use existing snippet if available, otherwise proceed with cloning + + if let Some(snippet_name) = existing_snippet_path { + println!("using existing snippet path skipping cloning"); + let snippet_path = snippets_dir.join(snippet_name); + let file_content_with_ident = extract_item_from_file( + &snippet_path, + &args.item_ident.as_ref().unwrap().to_string(), + )?; + let formatted_content = fix_indentation(&file_content_with_ident); + let output = into_example(&formatted_content, lang); + println!( + "embed_internal_str ----> Final output length: {}", + output.len() + ); + return Ok(output); + } // returns the directory of the cloned repo with caching + let repo_dir = clone_and_checkout_repo( git_url.value().as_str(), &new_snippet.commit_hash.as_ref().unwrap(), From df82e6c7d415a946b0a67426c2c11702f01a2005 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sat, 2 Nov 2024 00:33:04 +0530 Subject: [PATCH 29/55] moved creating of snppets dir to mod.rs --- macros/src/lib.rs | 14 +------------- macros/src/utils/mod.rs | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index cafe1bf..38bf5e0 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1181,19 +1181,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - println!("\nšŸ” Checking for existing snippets..."); - // creating snippets dir if it doesn't exist - let snippets_dir = caller_crate_root() - .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))? - .join(".snippets"); - println!("Using snippets directory: {}", snippets_dir.display()); - - fs::create_dir_all(&snippets_dir).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create snippets directory: {}", e), - ) - })?; - + let snippets_dir = get_or_create_snippets_dir()?; // Check for existing snippet let existing_snippet_path = if let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) { diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 36add42..5786132 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -9,7 +9,7 @@ use syn::visit::Visit; use syn::Error; use syn::Result; -use crate::{source_excerpt, ItemVisitor}; +use crate::{caller_crate_root, source_excerpt, ItemVisitor}; pub fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { let snippets_dir = crate_root.join(".snippets"); @@ -566,3 +566,24 @@ fn hash_git_url(url: &str) -> String { println!("ā„¹ļø Hashing git URL: {}", url); hash_string(url) } + +/// Creates and returns the snippets directory path, ensuring it exists +pub fn get_or_create_snippets_dir() -> Result { + let crate_root = caller_crate_root() + .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; + + let snippets_dir = crate_root.join(".snippets"); + println!( + "šŸ“ Ensuring snippets directory exists at: {}", + snippets_dir.display() + ); + + fs::create_dir_all(&snippets_dir).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to create .snippets directory: {}", e), + ) + })?; + + Ok(snippets_dir) +} From 73ab28a897c428c669c50876211e581da48790e0 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sat, 2 Nov 2024 02:02:57 +0530 Subject: [PATCH 30/55] added edge case where commit was provided and internet not available , so in that case created snippet path without commit --- macros/src/lib.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 38bf5e0..8bd5ad3 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1144,11 +1144,20 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - // need to add a check here if internet is not present but commit hash is provided // Create snippet file object based on connectivity - let new_snippet = if has_internet { + let new_snippet = if let Some(hash) = &args.commit_hash { + // If commit hash is provided, use it regardless of internet connectivity + println!("Using provided commit hash: {}", hash.value()); + SnippetFile::new_with_commit( + git_url.value().as_str(), + "commit", + &hash.value(), + &args.file_path.value(), + &hash.value(), + ) + } else if has_internet { + // Online mode without commit hash println!("\nšŸ” Fetching latest commit SHA from remote repository..."); - let commit_sha = if let Some(hash) = &args.commit_hash { - hash.value() - } else if let Some(tag) = &args.tag_name { + let commit_sha = if let Some(tag) = &args.tag_name { get_remote_commit_sha_without_clone( git_url.value().as_str(), None, @@ -1171,6 +1180,8 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - &commit_sha, ) } else { + // Offline mode without commit hash + println!("šŸ“” Offline mode: Creating snippet without commit hash"); SnippetFile::new_without_commit( git_url.value().as_str(), &git_option_type, From fbbc82301aaf6334266085b6f8c6b89264c0b1f0 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sat, 2 Nov 2024 02:48:06 +0530 Subject: [PATCH 31/55] hash of varied length working in case branch is not provided --- macros/src/lib.rs | 129 +++++++++++++++++++++++----------------- macros/src/utils/mod.rs | 39 ++++++++++++ 2 files changed, 112 insertions(+), 56 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 8bd5ad3..de8c609 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1101,46 +1101,32 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(git_url) = &args.git_url { let has_internet = check_internet_connectivity(); - // Determine git option type and value println!( "\nšŸŒ Internet connectivity: {}", if has_internet { "Online" } else { "Offline" } ); + // Determine git option type and value let (git_option_type, git_option_value) = if let Some(hash) = &args.commit_hash { println!("Using commit hash: {}", hash.value()); - ("commit".to_string(), hash.value()) + (Some("commit".to_string()), Some(hash.value())) } else if let Some(tag) = &args.tag_name { println!("Using tag: {}", tag.value()); - ("tag".to_string(), tag.value()) + (Some("tag".to_string()), Some(tag.value())) + } else if let Some(ref branch) = args.branch_name { + println!("Using provided branch: {}", branch.value()); + (Some("branch".to_string()), Some(branch.value())) } else { - // Branch case: Use provided branch or fallback - let branch_name = if let Some(ref branch) = args.branch_name { - println!("Using provided branch: {}", branch.value()); - branch.value() - } else if has_internet { - // No branch provided and online: try to get default branch - match get_default_branch(git_url.value().as_str()) { - Ok(branch) => { - println!("Using default branch: {}", branch); - branch - } - Err(_) => { - println!("Failed to get default branch, falling back to master"); - "master".to_string() - } - } - } else { - println!("Offline mode, using master as fallback"); - "master".to_string() - }; - ("branch".to_string(), branch_name) + // No specific git option provided - default branch case + println!("No specific git option provided, using flexible naming"); + (None, None) }; - println!( - "Git option type: {}, value: {}", - git_option_type, git_option_value - ); + if let (Some(opt_type), Some(opt_value)) = (&git_option_type, &git_option_value) { + println!("Git option type: {}, value: {}", opt_type, opt_value); + } else { + println!("Using flexible naming without git options"); + } // need to add a check here if internet is not present but commit hash is provided // Create snippet file object based on connectivity @@ -1154,45 +1140,76 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - &args.file_path.value(), &hash.value(), ) - } else if has_internet { - // Online mode without commit hash - println!("\nšŸ” Fetching latest commit SHA from remote repository..."); - let commit_sha = if let Some(tag) = &args.tag_name { - get_remote_commit_sha_without_clone( + } else if let Some(tag) = &args.tag_name { + // Tag case + if has_internet { + let commit_sha = get_remote_commit_sha_without_clone( git_url.value().as_str(), None, Some(tag.value().as_str()), - )? + )?; + SnippetFile::new_with_commit( + git_url.value().as_str(), + "tag", + &tag.value(), + &args.file_path.value(), + &commit_sha, + ) } else { - get_remote_commit_sha_without_clone( + println!("šŸ“” Offline mode: Creating snippet without commit hash for tag"); + SnippetFile::new_without_commit( git_url.value().as_str(), - Some(git_option_value.as_str()), + "tag", + &tag.value(), + &args.file_path.value(), + ) + } + } else if let Some(branch) = &args.branch_name { + // Explicit branch case + if has_internet { + let commit_sha = get_remote_commit_sha_without_clone( + git_url.value().as_str(), + Some(branch.value().as_str()), None, - )? - }; - println!("āœ… Found commit SHA: {}", commit_sha); - - SnippetFile::new_with_commit( - git_url.value().as_str(), - &git_option_type, - &git_option_value, - &args.file_path.value(), - &commit_sha, - ) + )?; + SnippetFile::new_with_commit( + git_url.value().as_str(), + "branch", + branch.value().as_str(), + &args.file_path.value(), + &commit_sha, + ) + } else { + println!("šŸ“” Offline mode: Creating snippet without commit hash for branch"); + SnippetFile::new_without_commit( + git_url.value().as_str(), + "branch", + branch.value().as_str(), + &args.file_path.value(), + ) + } } else { - // Offline mode without commit hash - println!("šŸ“” Offline mode: Creating snippet without commit hash"); - SnippetFile::new_without_commit( - git_url.value().as_str(), - &git_option_type, - &git_option_value, - &args.file_path.value(), - ) + // Default branch case - more flexible naming + if has_internet { + let commit_sha = + get_remote_commit_sha_without_clone(git_url.value().as_str(), None, None)?; + SnippetFile::new_for_default_branch( + git_url.value().as_str(), + &args.file_path.value(), + Some(&commit_sha), + ) + } else { + SnippetFile::new_for_default_branch( + git_url.value().as_str(), + &args.file_path.value(), + None, + ) + } }; - println!("\nšŸ” Checking for existing snippets..."); let snippets_dir = get_or_create_snippets_dir()?; + // Check for existing snippet let existing_snippet_path = if let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) { diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 5786132..ba8339a 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -500,6 +500,45 @@ impl SnippetFile { } }) } + + /// Creates a new SnippetFile for default branch case (when no branch is specified) + pub fn new_for_default_branch(git_url: &str, path: &str, commit_sha: Option<&str>) -> Self { + println!("\nšŸ“ Creating new SnippetFile for default branch case"); + println!("ā„¹ļø Input path: {}", path); + + let path_buf = PathBuf::from(path); + let file_name = path_buf + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or("unknown"); + println!("ā„¹ļø Extracted filename: {}", file_name); + + // Create prefix without git option hash for flexibility + let prefix = format!( + "{}-{}-{}", + hash_git_url(git_url), + hash_string(path), + file_name, + ); + println!("ā„¹ļø Generated prefix: {}", prefix); + + if let Some(commit) = commit_sha { + let full_name = format!("{}-{}.rs", prefix, commit); + println!("āœ… Created snippet filename with commit: {}", full_name); + Self { + prefix, + commit_hash: Some(commit.to_string()), + full_name, + } + } else { + println!("āœ… Created snippet filename without commit"); + Self { + prefix: prefix.clone(), + commit_hash: None, + full_name: prefix, + } + } + } } fn hash_string(input: &str) -> String { From e78619c1c516c0d4dcb1ecfa884a40f02fd5b18c Mon Sep 17 00:00:00 2001 From: Prakash Date: Sat, 2 Nov 2024 21:20:30 +0530 Subject: [PATCH 32/55] added ident and tested --- macros/src/lib.rs | 61 +++++++++++++++++++++++++++++------------ macros/src/utils/mod.rs | 55 ++++++++++++++++++++----------------- 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index de8c609..e8801db 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1139,6 +1139,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - &hash.value(), &args.file_path.value(), &hash.value(), + &args.item_ident.as_ref().unwrap().to_string(), ) } else if let Some(tag) = &args.tag_name { // Tag case @@ -1154,6 +1155,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - &tag.value(), &args.file_path.value(), &commit_sha, + &args.item_ident.as_ref().unwrap().to_string(), ) } else { println!("šŸ“” Offline mode: Creating snippet without commit hash for tag"); @@ -1162,6 +1164,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - "tag", &tag.value(), &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), ) } } else if let Some(branch) = &args.branch_name { @@ -1178,6 +1181,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - branch.value().as_str(), &args.file_path.value(), &commit_sha, + &args.item_ident.as_ref().unwrap().to_string(), ) } else { println!("šŸ“” Offline mode: Creating snippet without commit hash for branch"); @@ -1186,6 +1190,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - "branch", branch.value().as_str(), &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), ) } } else { @@ -1196,12 +1201,14 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - SnippetFile::new_for_default_branch( git_url.value().as_str(), &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), Some(&commit_sha), ) } else { SnippetFile::new_for_default_branch( git_url.value().as_str(), &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), None, ) } @@ -1266,11 +1273,21 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(snippet_name) = existing_snippet_path { println!("using existing snippet path skipping cloning"); let snippet_path = snippets_dir.join(snippet_name); - let file_content_with_ident = extract_item_from_file( - &snippet_path, - &args.item_ident.as_ref().unwrap().to_string(), - )?; - let formatted_content = fix_indentation(&file_content_with_ident); + // let file_content_with_ident = extract_item_from_file( + // &snippet_path, + // &args.item_ident.as_ref().unwrap().to_string(), + // )?; + let content = fs::read_to_string(&snippet_path).map_err(|e| { + Error::new( + args.file_path.span(), + format!( + "Failed to read snippet file: {} at path: {}", + e, + snippet_path.display() + ), + ) + })?; + let formatted_content = fix_indentation(&content); let output = into_example(&formatted_content, lang); println!( "embed_internal_str ----> Final output length: {}", @@ -1290,35 +1307,45 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let source_path = repo_dir.join(&args.file_path.value()); println!("Reading source file from: {}", source_path.display()); - let content = fs::read_to_string(&source_path).map_err(|e| { - Error::new( - args.file_path.span(), - format!("Failed to read file from repo: {}", e), - ) - })?; + let extracted_content: String = + extract_item_from_file(&source_path, &args.item_ident.as_ref().unwrap().to_string())?; + + println!( + "extracted_content: from extract_item_from_file using ident ----> {} \n\n extracted content {}", + args.item_ident.as_ref().unwrap().to_string(), + extracted_content + ); + + // let content = fs::read_to_string(&source_path).map_err(|e| { + // Error::new( + // args.file_path.span(), + // format!("Failed to read file from repo: {}", e), + // ) + // })?; let snippet_path = snippets_dir.join(&new_snippet.full_name); println!( "Writing content to snippet file: {}", snippet_path.display() ); - fs::write(&snippet_path, content).map_err(|e| { + fs::write(&snippet_path, extracted_content.clone()).map_err(|e| { Error::new( Span::call_site(), format!("Failed to write snippet file: {}", e), ) })?; + println!( "āœ… Wrote content to snippet file at path: {}", snippet_path.display() ); - let file_content_with_ident = extract_item_from_file( - &snippet_path, - &args.item_ident.as_ref().unwrap().to_string(), - )?; + // let file_content_with_ident = extract_item_from_file( + // &snippet_path, + // &args.item_ident.as_ref().unwrap().to_string(), + // )?; - let formatted_content = fix_indentation(&file_content_with_ident); + let formatted_content = fix_indentation(&extracted_content); let output = into_example(&formatted_content, lang); println!( "embed_internal_str ----> Final output length: {}", diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index ba8339a..2613747 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -399,30 +399,25 @@ impl SnippetFile { git_option_type: &str, git_option_value: &str, path: &str, + item_ident: &str, ) -> Self { - println!("\nšŸ“ Creating new SnippetFile.. name without commit."); + println!("\nšŸ“ Creating new SnippetFile..."); println!("ā„¹ļø Input path: {}", path); - - let path_buf = PathBuf::from(path); - let file_name = path_buf - .file_name() - .and_then(|f| f.to_str()) - .unwrap_or("unknown"); - println!("ā„¹ļø Extracted filename: {}", file_name); + println!("ā„¹ļø Item identifier: {}", item_ident); let prefix = format!( "{}-{}-{}-{}", hash_git_url(git_url), hash_git_option(git_option_type, git_option_value), hash_string(path), - file_name, + item_ident, ); println!("ā„¹ļø Generated prefix: {}", prefix); Self { prefix: prefix.clone(), commit_hash: None, - full_name: prefix, + full_name: format!("{}.rs", prefix), } } @@ -432,14 +427,26 @@ impl SnippetFile { git_option_value: &str, path: &str, commit_sha: &str, + item_ident: &str, ) -> Self { - println!("\nšŸ“ Creating new SnippetFile.. name with commit."); - let base = Self::new_without_commit(git_url, git_option_type, git_option_value, path); - let full_name = format!("{}-{}.rs", base.prefix, commit_sha); + println!("\nšŸ“ Creating new SnippetFile..."); + println!("ā„¹ļø Input path: {}", path); + println!("ā„¹ļø Item identifier: {}", item_ident); + + let prefix = format!( + "{}-{}-{}-{}", + hash_git_url(git_url), + hash_git_option(git_option_type, git_option_value), + hash_string(path), + item_ident, + ); + println!("ā„¹ļø Generated prefix: {}", prefix); + + let full_name = format!("{}-{}.rs", prefix, commit_sha); println!("āœ… Created snippet filename: {}", full_name); Self { - prefix: base.prefix, + prefix, commit_hash: Some(commit_sha.to_string()), full_name, } @@ -502,23 +509,21 @@ impl SnippetFile { } /// Creates a new SnippetFile for default branch case (when no branch is specified) - pub fn new_for_default_branch(git_url: &str, path: &str, commit_sha: Option<&str>) -> Self { + pub fn new_for_default_branch( + git_url: &str, + path: &str, + item_ident: &str, + commit_sha: Option<&str>, + ) -> Self { println!("\nšŸ“ Creating new SnippetFile for default branch case"); println!("ā„¹ļø Input path: {}", path); + println!("ā„¹ļø Item identifier: {}", item_ident); - let path_buf = PathBuf::from(path); - let file_name = path_buf - .file_name() - .and_then(|f| f.to_str()) - .unwrap_or("unknown"); - println!("ā„¹ļø Extracted filename: {}", file_name); - - // Create prefix without git option hash for flexibility let prefix = format!( "{}-{}-{}", hash_git_url(git_url), hash_string(path), - file_name, + item_ident, ); println!("ā„¹ļø Generated prefix: {}", prefix); @@ -535,7 +540,7 @@ impl SnippetFile { Self { prefix: prefix.clone(), commit_hash: None, - full_name: prefix, + full_name: format!("{}.rs", prefix), } } } From 1b3f34b3a31791b41158621918b570a844259c57 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 01:15:26 +0530 Subject: [PATCH 33/55] changed parse for embed args , cleanup and addeed tests , getting stack overflow error on the test tests::embed_args_tests::test_token_stream_conversion_git --- macros/src/lib.rs | 344 +++++++++++++++++++++++++++++----------- macros/src/tests.rs | 265 +++++++++++++++++++++++++++++++ macros/src/utils/mod.rs | 163 ------------------- 3 files changed, 516 insertions(+), 256 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index e8801db..02ffd3f 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -553,6 +553,7 @@ mod kw { syn::custom_keyword!(tag); syn::custom_keyword!(item); } + struct EmbedArgs { git_url: Option, file_path: LitStr, @@ -562,88 +563,70 @@ struct EmbedArgs { item_ident: Option, } -impl Parse for EmbedArgs { - fn parse(input: ParseStream) -> Result { - // First try to parse as positional arguments - if !input.peek(kw::git) && !input.peek(kw::path) { - let file_path: LitStr = input.parse()?; - - // Validate that file_path is not a URL - if file_path.value().starts_with("http://") || file_path.value().starts_with("https://") - { - return Err(Error::new( - file_path.span(), - "First positional argument must be a file path, not a URL. For git URLs, use named arguments: git: \"url\", path: \"file_path\"", - )); - } - - let item_ident = if input.peek(Token![,]) { - input.parse::()?; - Some(input.parse()?) - } else { - None - }; - - return Ok(EmbedArgs { - git_url: None, - file_path, - branch_name: None, - commit_hash: None, - tag_name: None, - item_ident, - }); - } +// Manually implement Debug +impl std::fmt::Debug for EmbedArgs { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EmbedArgs") + .field("git_url", &self.git_url.as_ref().map(|s| s.value())) + .field("file_path", &self.file_path.value()) + .field("branch_name", &self.branch_name.as_ref().map(|s| s.value())) + .field("commit_hash", &self.commit_hash.as_ref().map(|s| s.value())) + .field("tag_name", &self.tag_name.as_ref().map(|s| s.value())) + .field( + "item_ident", + &self.item_ident.as_ref().map(|i| i.to_string()), + ) + .finish() + } +} - // If not positional, parse as named arguments - let mut git_url = None; - let mut file_path = None; - let mut branch_name = None; - let mut commit_hash = None; - let mut tag_name = None; - let mut item_ident = None; +impl EmbedArgs { + /// Creates a new EmbedArgs instance with validation + fn new( + git_url: Option, + file_path: LitStr, + branch_name: Option, + commit_hash: Option, + tag_name: Option, + item_ident: Option, + ) -> Result { + let args = Self { + git_url, + file_path, + branch_name, + commit_hash, + tag_name, + item_ident, + }; + args.validate()?; + Ok(args) + } - while !input.is_empty() { - let lookahead = input.lookahead1(); - - if lookahead.peek(kw::git) { - let _: kw::git = input.parse()?; - let _: Token![:] = input.parse()?; - git_url = Some(input.parse()?); - } else if lookahead.peek(kw::path) { - let _: kw::path = input.parse()?; - let _: Token![:] = input.parse()?; - file_path = Some(input.parse()?); - } else if lookahead.peek(kw::branch) { - let _: kw::branch = input.parse()?; - let _: Token![:] = input.parse()?; - branch_name = Some(input.parse()?); - } else if lookahead.peek(kw::commit) { - let _: kw::commit = input.parse()?; - let _: Token![:] = input.parse()?; - commit_hash = Some(input.parse()?); - } else if lookahead.peek(kw::tag) { - let _: kw::tag = input.parse()?; - let _: Token![:] = input.parse()?; - tag_name = Some(input.parse()?); - } else if lookahead.peek(kw::item) { - let _: kw::item = input.parse()?; - let _: Token![:] = input.parse()?; - item_ident = Some(input.parse()?); - } else { - return Err(lookahead.error()); - } + /// Validates all argument constraints + fn validate(&self) -> Result<()> { + self.validate_file_path()?; + self.validate_git_refs()?; + self.validate_git_dependencies()?; + Ok(()) + } - if !input.is_empty() { - let _: Token![,] = input.parse()?; - } + /// Ensures file path is valid based on context + fn validate_file_path(&self) -> Result<()> { + if self.git_url.is_none() + && (self.file_path.value().starts_with("http://") + || self.file_path.value().starts_with("https://")) + { + return Err(Error::new( + self.file_path.span(), + "File path cannot be a URL. Use git: \"url\" for git repositories", + )); } + Ok(()) + } - // Validate required parameters - let file_path = - file_path.ok_or_else(|| Error::new(Span::call_site(), "path parameter is required"))?; - - // Validate that only one of branch, commit, or tag is specified - let ref_count = [&branch_name, &commit_hash, &tag_name] + /// Ensures only one git reference type is specified + fn validate_git_refs(&self) -> Result<()> { + let ref_count = [&self.branch_name, &self.commit_hash, &self.tag_name] .iter() .filter(|&&x| x.is_some()) .count(); @@ -654,36 +637,211 @@ impl Parse for EmbedArgs { "Only one of branch, commit, or tag can be specified", )); } + Ok(()) + } - // Validate that branch, commit, or tag are only used with git parameter - if git_url.is_none() - && (branch_name.is_some() || commit_hash.is_some() || tag_name.is_some()) + /// Ensures git-specific arguments are only used with git URLs + fn validate_git_dependencies(&self) -> Result<()> { + if self.git_url.is_none() + && (self.branch_name.is_some() || self.commit_hash.is_some() || self.tag_name.is_some()) { return Err(Error::new( Span::call_site(), - "branch, commit, or tag can only be used when git parameter is specified", + "branch, commit, or tag can only be used with git parameter", )); } + Ok(()) + } - Ok(EmbedArgs { - git_url, + /// Parses positional arguments format + fn parse_positional(input: ParseStream) -> Result { + let file_path: LitStr = input.parse()?; + + let item_ident = if input.peek(Token![,]) { + input.parse::()?; + Some(input.parse()?) + } else { + None + }; + + Self::new(None, file_path, None, None, None, item_ident) + } + + /// Parses named arguments format + fn parse_named(input: ParseStream) -> Result { + let mut builder = NamedArgsBuilder::new(); + + while !input.is_empty() { + builder.set_arg(input)?; + } + + builder.build() + } +} + +/// Builder for collecting named arguments during parsing +#[derive(Default)] +struct NamedArgsBuilder { + git_url: Option, + file_path: Option, + branch_name: Option, + commit_hash: Option, + tag_name: Option, + item_ident: Option, +} + +impl NamedArgsBuilder { + fn new() -> Self { + Self::default() + } + + fn set_arg(&mut self, input: ParseStream) -> Result<()> { + let lookahead = input.lookahead1(); + + if lookahead.peek(kw::git) { + self.parse_git_url(input)?; + } else if lookahead.peek(kw::path) { + self.parse_file_path(input)?; + } else if lookahead.peek(kw::branch) { + self.parse_branch(input)?; + } else if lookahead.peek(kw::commit) { + self.parse_commit(input)?; + } else if lookahead.peek(kw::tag) { + self.parse_tag(input)?; + } else if lookahead.peek(kw::item) { + self.parse_item(input)?; + } else { + return Err(lookahead.error()); + } + + if !input.is_empty() { + input.parse::()?; + } + Ok(()) + } + + fn parse_git_url(&mut self, input: ParseStream) -> Result<()> { + let _: kw::git = input.parse()?; + let _: Token![:] = input.parse()?; + self.git_url = Some(input.parse()?); + Ok(()) + } + + fn parse_file_path(&mut self, input: ParseStream) -> Result<()> { + let _: kw::path = input.parse()?; + let _: Token![:] = input.parse()?; + self.file_path = Some(input.parse()?); + Ok(()) + } + + fn parse_branch(&mut self, input: ParseStream) -> Result<()> { + let _: kw::branch = input.parse()?; + let _: Token![:] = input.parse()?; + self.branch_name = Some(input.parse()?); + Ok(()) + } + + fn parse_commit(&mut self, input: ParseStream) -> Result<()> { + let _: kw::commit = input.parse()?; + let _: Token![:] = input.parse()?; + self.commit_hash = Some(input.parse()?); + Ok(()) + } + + fn parse_tag(&mut self, input: ParseStream) -> Result<()> { + let _: kw::tag = input.parse()?; + let _: Token![:] = input.parse()?; + self.tag_name = Some(input.parse()?); + Ok(()) + } + + fn parse_item(&mut self, input: ParseStream) -> Result<()> { + let _: kw::item = input.parse()?; + let _: Token![:] = input.parse()?; + self.item_ident = Some(input.parse()?); + Ok(()) + } + + fn build(self) -> Result { + let file_path = self + .file_path + .ok_or_else(|| Error::new(Span::call_site(), "path parameter is required"))?; + + EmbedArgs::new( + self.git_url, file_path, - branch_name, - commit_hash, - tag_name, - item_ident, - }) + self.branch_name, + self.commit_hash, + self.tag_name, + self.item_ident, + ) + } +} + +impl Parse for EmbedArgs { + fn parse(input: ParseStream) -> Result { + if !input.peek(kw::git) && !input.peek(kw::path) { + return Self::parse_positional(input); + } + Self::parse_named(input) } } impl ToTokens for EmbedArgs { fn to_tokens(&self, tokens: &mut TokenStream2) { - tokens.extend(self.file_path.to_token_stream()); - let Some(item_ident) = &self.item_ident else { + // For positional arguments style + if self.git_url.is_none() && !self.has_named_args() { + self.file_path.to_tokens(tokens); + if let Some(ref item) = self.item_ident { + Token![,](Span::call_site()).to_tokens(tokens); + item.to_tokens(tokens); + } return; - }; - tokens.extend(quote!(,)); - tokens.extend(item_ident.to_token_stream()); + } + + // For named arguments style + let mut args = TokenStream2::new(); + + if let Some(ref git) = self.git_url { + quote!(git: #git,).to_tokens(&mut args); + } + + quote!(path: #self.file_path,).to_tokens(&mut args); + + if let Some(ref branch) = self.branch_name { + quote!(branch: #branch,).to_tokens(&mut args); + } + + if let Some(ref commit) = self.commit_hash { + quote!(commit: #commit,).to_tokens(&mut args); + } + + if let Some(ref tag) = self.tag_name { + quote!(tag: #tag,).to_tokens(&mut args); + } + + if let Some(ref item) = self.item_ident { + quote!(item: #item,).to_tokens(&mut args); + } + + tokens.extend(args); + } +} + +// Add this helper method to EmbedArgs impl +impl EmbedArgs { + fn has_named_args(&self) -> bool { + self.branch_name.is_some() + || self.commit_hash.is_some() + || self.tag_name.is_some() + || self.git_url.is_some() + } + + // Add this method to convert to TokenStream2 + pub fn to_token_stream(&self) -> TokenStream2 { + let mut tokens = TokenStream2::new(); + self.to_tokens(&mut tokens); + tokens } } diff --git a/macros/src/tests.rs b/macros/src/tests.rs index d7b7029..d08e043 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -1,4 +1,7 @@ use super::*; +use std::fs; +use std::path::PathBuf; +use tempfile::TempDir; #[test] fn test_export_basic_parsing_valid() { @@ -193,3 +196,265 @@ fn bar() { "#; assert_eq!(fix_leading_indentation(input), output); } + +#[cfg(test)] +mod embed_args_tests { + use super::*; + use quote::quote; + use syn::parse2; + + // Fixed helper function - using syn::Result instead of std::result::Result + fn assert_error_contains(result: syn::Result, expected: &str) { + match result { + Ok(_) => panic!("Expected error, got success"), + Err(e) => assert!( + e.to_string().contains(expected), + "Error message '{}' should contain '{}'", + e.to_string(), + expected + ), + } + } + + #[test] + fn test_basic_positional_args() { + // Test: Basic file path only + let input = quote!("src/lib.rs"); + let args = parse2::(input.clone()).unwrap(); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert!(args.item_ident.is_none()); + + // Test: File path with item identifier + let input = quote!("src/lib.rs", my_function); + let args = parse2::(input.clone()).unwrap(); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); + } + + #[test] + fn test_named_args_basic() { + let input = quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main" + ); + let args = parse2::(input).unwrap(); + + assert_eq!( + args.git_url.unwrap().value(), + "https://github.com/user/repo" + ); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert_eq!(args.branch_name.unwrap().value(), "main"); + } + + #[test] + fn test_named_args_all_fields() { + let input = quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main", + item: my_function + ); + let args = parse2::(input).unwrap(); + + assert!(args.git_url.is_some()); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert_eq!(args.branch_name.unwrap().value(), "main"); + assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); + } + + #[test] + fn test_error_missing_path() { + // Test: Named args without path + let input = quote!( + git: "https://github.com/user/repo", + branch: "main" + ); + assert_error_contains(parse2::(input), "path parameter is required"); + } + + #[test] + fn test_error_multiple_git_refs() { + // Test: Multiple git references (branch and tag) + let input = quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main", + tag: "v1.0" + ); + assert_error_contains( + parse2::(input), + "Only one of branch, commit, or tag can be specified", + ); + + // Test: Multiple git references (branch and commit) + let input = quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main", + commit: "abc123" + ); + assert_error_contains( + parse2::(input), + "Only one of branch, commit, or tag can be specified", + ); + } + + #[test] + fn test_error_git_refs_without_git_url() { + // Test: Branch without git URL + let input = quote!( + path: "src/lib.rs", + branch: "main" + ); + assert_error_contains( + parse2::(input), + "branch, commit, or tag can only be used with git parameter", + ); + + // Test: Tag without git URL + let input = quote!( + path: "src/lib.rs", + tag: "v1.0" + ); + assert_error_contains( + parse2::(input), + "branch, commit, or tag can only be used with git parameter", + ); + } + + #[test] + fn test_error_invalid_url_in_path() { + // Test: URL in file path for positional args + let input = quote!("https://github.com/user/repo/file.rs"); + assert_error_contains(parse2::(input), "File path cannot be a URL"); + } + + #[test] + fn test_error_invalid_syntax() { + // Test: Invalid token in positional args + let input = quote!("src/lib.rs" something); + let result = parse2::(input); + assert!(result.is_err()); + + // Test: Invalid named argument + let input = quote!( + git: "https://github.com/user/repo", + invalid: "value", + path: "src/lib.rs" + ); + let result = parse2::(input); + assert!(result.is_err()); + } + + #[test] + fn test_token_stream_conversion_positional() { + // Test positional args only + let input = quote!("src/lib.rs", my_function); + let args = parse2::(input).unwrap(); + + // Convert to token stream + let mut tokens = TokenStream2::new(); + args.to_tokens(&mut tokens); + + // Convert to string carefully + let tokens_str = tokens + .into_iter() + .map(|t| t.to_string()) + .collect::>() + .join(" "); + + println!("Positional tokens: {}", tokens_str); + assert!(tokens_str.contains("src/lib.rs")); + assert!(tokens_str.contains("my_function")); + } + + #[test] + fn test_token_stream_conversion_named() { + // Test named args only + let input = quote!( + path: "src/lib.rs" + ); + let args = parse2::(input).unwrap(); + + // Convert to token stream + let mut tokens = TokenStream2::new(); + args.to_tokens(&mut tokens); + + // Convert to string carefully + let tokens_str = tokens + .into_iter() + .map(|t| t.to_string()) + .collect::>() + .join(" "); + + println!("Named tokens: {}", tokens_str); + assert!(tokens_str.contains("path")); + assert!(tokens_str.contains("src/lib.rs")); + } + + #[test] + fn test_token_stream_conversion_git() { + // Test git args + let input = quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main" + ); + let args = parse2::(input).unwrap(); + + // Convert to token stream + let mut tokens = TokenStream2::new(); + args.to_tokens(&mut tokens); + + // Convert to string carefully + let tokens_str = tokens + .into_iter() + .map(|t| t.to_string()) + .collect::>() + .join(" "); + + println!("Git tokens: {}", tokens_str); + assert!(tokens_str.contains("git")); + assert!(tokens_str.contains("https://github.com/user/repo")); + assert!(tokens_str.contains("path")); + assert!(tokens_str.contains("src/lib.rs")); + assert!(tokens_str.contains("branch")); + assert!(tokens_str.contains("main")); + } + + #[test] + fn test_whitespace_handling() { + // Test: Extra whitespace in positional args + let input = quote!("src/lib.rs", my_function); + let result = parse2::(input); + assert!(result.is_ok()); + + // Test: Extra whitespace in named args + let input = quote!( + git : "https://github.com/user/repo" , + path : "src/lib.rs" , + branch : "main" + ); + let result = parse2::(input); + assert!(result.is_ok()); + } + + #[test] + fn test_empty_string_values() { + // Test: Empty file path + let input = quote!(""); + let result = parse2::(input); + assert!(result.is_ok()); // Empty string is valid, but might fail later validation + + // Test: Empty values in named args + let input = quote!( + git: "", + path: "", + branch: "" + ); + let result = parse2::(input); + assert!(result.is_ok()); // Syntactically valid, but might fail later validation + } +} diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 2613747..2987a84 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -11,96 +11,6 @@ use syn::Result; use crate::{caller_crate_root, source_excerpt, ItemVisitor}; -pub fn manage_snippet(crate_root: &Path, file_path: &str, item_ident: &str) -> Result { - let snippets_dir = crate_root.join(".snippets"); - println!("Snippets directory: {}", snippets_dir.display()); - - // Ensure snippets directory exists - fs::create_dir_all(&snippets_dir).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create .snippets directory: {}", e), - ) - })?; - - let snippet_path = generate_snippet_path(&snippets_dir, file_path, item_ident); - println!("Snippet path: {}", snippet_path.display()); - - let has_internet = check_internet_connectivity(); - println!( - "Internet connectivity: {}", - if has_internet { - "Available" - } else { - "Not available" - } - ); - - if !has_internet { - println!("No internet connection, attempting to read from cached snippet"); - // Try to read from the snippet file - match fs::read_to_string(&snippet_path) { - Ok(content) => { - println!("Successfully read content from cached snippet"); - Ok(content) - } - Err(e) => Err(Error::new( - Span::call_site(), - format!( - "No internet connection and failed to read cached snippet at {}: {}. - Please ensure you have internet connectivity for the first run.", - snippet_path.display(), - e - ), - )), - } - } else { - // Internet is available, proceed with full path checking and content updating - let full_path = crate_root.join(file_path); - println!( - "Internet available, checking file at: {}", - full_path.display() - ); - - let existing_content = fs::read_to_string(&snippet_path).ok(); - let new_content = extract_item_from_file(&full_path, item_ident)?; - - if existing_content.as_ref().map(|c| hash_content(c)) != Some(hash_content(&new_content)) { - println!("Updating snippet file with new content"); - fs::write(&snippet_path, &new_content).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to write snippet file: {}", e), - ) - })?; - } else { - println!("Snippet is up to date"); - } - - Ok(new_content) - } -} - -pub fn generate_snippet_path(snippets_dir: &Path, file_path: &str, item_ident: &str) -> PathBuf { - println!( - "inside generate_snippet_path ----> Snippets directory: {}", - snippets_dir.display() - ); - println!( - "inside generate_snippet_path ----> File path: {}", - file_path - ); - println!( - "inside generate_snippet_path ----> Item ident: {}", - item_ident - ); - let path = PathBuf::from(file_path); - let file_name = path - .file_name() - .and_then(|f| f.to_str()) - .unwrap_or("unknown"); - snippets_dir.join(format!(".docify-snippet-{}-{}", file_name, item_ident)) -} pub fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { println!( "inside extract_item_from_file ----> Extracting item '{}' from '{}'", @@ -149,14 +59,6 @@ pub fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result String { - let mut hasher = Sha256::new(); - hasher.update(content); - let result = format!("{:x}", hasher.finalize()); - println!("Content hash: {}", result); - result -} - /// Checks if there is an active internet connection by attempting to connect to multiple reliable hosts pub fn check_internet_connectivity() -> bool { // List of reliable hosts and ports to try @@ -368,23 +270,6 @@ pub fn clone_and_checkout_repo(git_url: &str, commit_sha: &str) -> Result String { - let path_buf = PathBuf::from(path); - let file_name = path_buf - .file_name() - .and_then(|f| f.to_str()) - .unwrap_or("unknown"); - - format!("{}-{}-{}", &commit_sha[..8], hash_path(path), file_name) -} - -fn hash_path(path: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(path.as_bytes()); - format!("{:.8x}", hasher.finalize()) // First 8 chars of hash -} - /// Represents a parsed snippet filename pub struct SnippetFile { pub prefix: String, @@ -558,54 +443,6 @@ fn hash_git_option(option_type: &str, value: &str) -> String { /// Helper function to get default branch -pub fn get_default_branch(git_url: &str) -> Result { - let temp_dir = tempfile::Builder::new() - .prefix("docify-temp-") - .rand_bytes(5) - .tempdir() - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create temp dir: {}", e), - ) - })?; - - let repo = Repository::init(temp_dir.path()) - .map_err(|e| Error::new(Span::call_site(), format!("Failed to init repo: {}", e)))?; - - let mut remote = repo - .remote_anonymous(git_url) - .map_err(|e| Error::new(Span::call_site(), format!("Failed to create remote: {}", e)))?; - - remote.connect(git2::Direction::Fetch).map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to connect to remote: {}", e), - ) - })?; - - let default_branch = remote - .default_branch() - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to get default branch: {}", e), - ) - })? - .as_str() - .ok_or_else(|| Error::new(Span::call_site(), "Invalid default branch name"))? - .to_string(); - - remote - .disconnect() - .map_err(|e| Error::new(Span::call_site(), format!("Failed to disconnect: {}", e)))?; - - Ok(default_branch - .strip_prefix("refs/heads/") - .unwrap_or(&default_branch) - .to_string()) -} - fn hash_git_url(url: &str) -> String { println!("ā„¹ļø Hashing git URL: {}", url); hash_string(url) From 8af20542add9dcd8663f4600941e42c87085c0a4 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 02:25:15 +0530 Subject: [PATCH 34/55] removed embed args test added logs in lib.rs --- macros/src/lib.rs | 10 ++ macros/src/tests.rs | 262 -------------------------------------------- 2 files changed, 10 insertions(+), 262 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 02ffd3f..a097404 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1307,6 +1307,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - None, Some(tag.value().as_str()), )?; + println!("commit hash for tag: {} -> {}", tag.value(), commit_sha); SnippetFile::new_with_commit( git_url.value().as_str(), "tag", @@ -1333,6 +1334,11 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - Some(branch.value().as_str()), None, )?; + println!( + "commit hash for branch: {} -> {}", + branch.value(), + commit_sha + ); SnippetFile::new_with_commit( git_url.value().as_str(), "branch", @@ -1356,6 +1362,10 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if has_internet { let commit_sha = get_remote_commit_sha_without_clone(git_url.value().as_str(), None, None)?; + println!( + "branch not provided getting latest commit hash for the default branch -> {}", + commit_sha + ); SnippetFile::new_for_default_branch( git_url.value().as_str(), &args.file_path.value(), diff --git a/macros/src/tests.rs b/macros/src/tests.rs index d08e043..cd926b2 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -196,265 +196,3 @@ fn bar() { "#; assert_eq!(fix_leading_indentation(input), output); } - -#[cfg(test)] -mod embed_args_tests { - use super::*; - use quote::quote; - use syn::parse2; - - // Fixed helper function - using syn::Result instead of std::result::Result - fn assert_error_contains(result: syn::Result, expected: &str) { - match result { - Ok(_) => panic!("Expected error, got success"), - Err(e) => assert!( - e.to_string().contains(expected), - "Error message '{}' should contain '{}'", - e.to_string(), - expected - ), - } - } - - #[test] - fn test_basic_positional_args() { - // Test: Basic file path only - let input = quote!("src/lib.rs"); - let args = parse2::(input.clone()).unwrap(); - assert_eq!(args.file_path.value(), "src/lib.rs"); - assert!(args.item_ident.is_none()); - - // Test: File path with item identifier - let input = quote!("src/lib.rs", my_function); - let args = parse2::(input.clone()).unwrap(); - assert_eq!(args.file_path.value(), "src/lib.rs"); - assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); - } - - #[test] - fn test_named_args_basic() { - let input = quote!( - git: "https://github.com/user/repo", - path: "src/lib.rs", - branch: "main" - ); - let args = parse2::(input).unwrap(); - - assert_eq!( - args.git_url.unwrap().value(), - "https://github.com/user/repo" - ); - assert_eq!(args.file_path.value(), "src/lib.rs"); - assert_eq!(args.branch_name.unwrap().value(), "main"); - } - - #[test] - fn test_named_args_all_fields() { - let input = quote!( - git: "https://github.com/user/repo", - path: "src/lib.rs", - branch: "main", - item: my_function - ); - let args = parse2::(input).unwrap(); - - assert!(args.git_url.is_some()); - assert_eq!(args.file_path.value(), "src/lib.rs"); - assert_eq!(args.branch_name.unwrap().value(), "main"); - assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); - } - - #[test] - fn test_error_missing_path() { - // Test: Named args without path - let input = quote!( - git: "https://github.com/user/repo", - branch: "main" - ); - assert_error_contains(parse2::(input), "path parameter is required"); - } - - #[test] - fn test_error_multiple_git_refs() { - // Test: Multiple git references (branch and tag) - let input = quote!( - git: "https://github.com/user/repo", - path: "src/lib.rs", - branch: "main", - tag: "v1.0" - ); - assert_error_contains( - parse2::(input), - "Only one of branch, commit, or tag can be specified", - ); - - // Test: Multiple git references (branch and commit) - let input = quote!( - git: "https://github.com/user/repo", - path: "src/lib.rs", - branch: "main", - commit: "abc123" - ); - assert_error_contains( - parse2::(input), - "Only one of branch, commit, or tag can be specified", - ); - } - - #[test] - fn test_error_git_refs_without_git_url() { - // Test: Branch without git URL - let input = quote!( - path: "src/lib.rs", - branch: "main" - ); - assert_error_contains( - parse2::(input), - "branch, commit, or tag can only be used with git parameter", - ); - - // Test: Tag without git URL - let input = quote!( - path: "src/lib.rs", - tag: "v1.0" - ); - assert_error_contains( - parse2::(input), - "branch, commit, or tag can only be used with git parameter", - ); - } - - #[test] - fn test_error_invalid_url_in_path() { - // Test: URL in file path for positional args - let input = quote!("https://github.com/user/repo/file.rs"); - assert_error_contains(parse2::(input), "File path cannot be a URL"); - } - - #[test] - fn test_error_invalid_syntax() { - // Test: Invalid token in positional args - let input = quote!("src/lib.rs" something); - let result = parse2::(input); - assert!(result.is_err()); - - // Test: Invalid named argument - let input = quote!( - git: "https://github.com/user/repo", - invalid: "value", - path: "src/lib.rs" - ); - let result = parse2::(input); - assert!(result.is_err()); - } - - #[test] - fn test_token_stream_conversion_positional() { - // Test positional args only - let input = quote!("src/lib.rs", my_function); - let args = parse2::(input).unwrap(); - - // Convert to token stream - let mut tokens = TokenStream2::new(); - args.to_tokens(&mut tokens); - - // Convert to string carefully - let tokens_str = tokens - .into_iter() - .map(|t| t.to_string()) - .collect::>() - .join(" "); - - println!("Positional tokens: {}", tokens_str); - assert!(tokens_str.contains("src/lib.rs")); - assert!(tokens_str.contains("my_function")); - } - - #[test] - fn test_token_stream_conversion_named() { - // Test named args only - let input = quote!( - path: "src/lib.rs" - ); - let args = parse2::(input).unwrap(); - - // Convert to token stream - let mut tokens = TokenStream2::new(); - args.to_tokens(&mut tokens); - - // Convert to string carefully - let tokens_str = tokens - .into_iter() - .map(|t| t.to_string()) - .collect::>() - .join(" "); - - println!("Named tokens: {}", tokens_str); - assert!(tokens_str.contains("path")); - assert!(tokens_str.contains("src/lib.rs")); - } - - #[test] - fn test_token_stream_conversion_git() { - // Test git args - let input = quote!( - git: "https://github.com/user/repo", - path: "src/lib.rs", - branch: "main" - ); - let args = parse2::(input).unwrap(); - - // Convert to token stream - let mut tokens = TokenStream2::new(); - args.to_tokens(&mut tokens); - - // Convert to string carefully - let tokens_str = tokens - .into_iter() - .map(|t| t.to_string()) - .collect::>() - .join(" "); - - println!("Git tokens: {}", tokens_str); - assert!(tokens_str.contains("git")); - assert!(tokens_str.contains("https://github.com/user/repo")); - assert!(tokens_str.contains("path")); - assert!(tokens_str.contains("src/lib.rs")); - assert!(tokens_str.contains("branch")); - assert!(tokens_str.contains("main")); - } - - #[test] - fn test_whitespace_handling() { - // Test: Extra whitespace in positional args - let input = quote!("src/lib.rs", my_function); - let result = parse2::(input); - assert!(result.is_ok()); - - // Test: Extra whitespace in named args - let input = quote!( - git : "https://github.com/user/repo" , - path : "src/lib.rs" , - branch : "main" - ); - let result = parse2::(input); - assert!(result.is_ok()); - } - - #[test] - fn test_empty_string_values() { - // Test: Empty file path - let input = quote!(""); - let result = parse2::(input); - assert!(result.is_ok()); // Empty string is valid, but might fail later validation - - // Test: Empty values in named args - let input = quote!( - git: "", - path: "", - branch: "" - ); - let result = parse2::(input); - assert!(result.is_ok()); // Syntactically valid, but might fail later validation - } -} From 7d17b26613339655477fe9e387d7ddada05ba67e Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 02:57:37 +0530 Subject: [PATCH 35/55] cleanup and added comments --- macros/src/lib.rs | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index a097404..cd3d23d 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1254,6 +1254,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - )); } + // get the root of the crate let crate_root = caller_crate_root() .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; @@ -1280,14 +1281,22 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - (None, None) }; + // print the git option type and value if let (Some(opt_type), Some(opt_value)) = (&git_option_type, &git_option_value) { println!("Git option type: {}, value: {}", opt_type, opt_value); } else { println!("Using flexible naming without git options"); } - // need to add a check here if internet is not present but commit hash is provided - // Create snippet file object based on connectivity + // if git option it present the name of the snippet will be of the format + // [git-url-hash]-[git-option-hash]-[path-hash]-[ident_name]-[full-commit-hash].rs + + // if no git option is present the name of the snippet will be of the format + // this will be applicable only when none of branch / commit / tag is provided + // [git-url-hash]-[path-hash]-[ident_name]-[full-commit-hash].rs + + // first create the snippet file name based on the passed arguments + let new_snippet = if let Some(hash) = &args.commit_hash { // If commit hash is provided, use it regardless of internet connectivity println!("Using provided commit hash: {}", hash.value()); @@ -1381,8 +1390,17 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ) } }; - println!("\nšŸ” Checking for existing snippets..."); + // snippet file name is created now check if it already exists + + println!( + "created snippet file name based on the passed arguments ----> {}", + new_snippet.full_name + ); + + println!("\nšŸ” Now Checking if the snippet file already exists"); + + // create the snippets directory if it doesn't exist let snippets_dir = get_or_create_snippets_dir()?; // Check for existing snippet @@ -1426,6 +1444,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } } } else if !has_internet { + println!("No matching snippet found and no internet connection available"); return Err(Error::new( Span::call_site(), "No matching snippet found and no internet connection available", @@ -1441,10 +1460,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(snippet_name) = existing_snippet_path { println!("using existing snippet path skipping cloning"); let snippet_path = snippets_dir.join(snippet_name); - // let file_content_with_ident = extract_item_from_file( - // &snippet_path, - // &args.item_ident.as_ref().unwrap().to_string(), - // )?; + let content = fs::read_to_string(&snippet_path).map_err(|e| { Error::new( args.file_path.span(), @@ -1484,13 +1500,6 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - extracted_content ); - // let content = fs::read_to_string(&source_path).map_err(|e| { - // Error::new( - // args.file_path.span(), - // format!("Failed to read file from repo: {}", e), - // ) - // })?; - let snippet_path = snippets_dir.join(&new_snippet.full_name); println!( "Writing content to snippet file: {}", @@ -1508,11 +1517,6 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - snippet_path.display() ); - // let file_content_with_ident = extract_item_from_file( - // &snippet_path, - // &args.item_ident.as_ref().unwrap().to_string(), - // )?; - let formatted_content = fix_indentation(&extracted_content); let output = into_example(&formatted_content, lang); println!( From 4cc75eaa5e9080b5b8b39f341c07fe0c57ffa988 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 03:20:26 +0530 Subject: [PATCH 36/55] encapsulated git options into a separate function --- macros/src/lib.rs | 18 ++++-------------- macros/src/utils/mod.rs | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index cd3d23d..8a6939e 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1266,20 +1266,10 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ); // Determine git option type and value - let (git_option_type, git_option_value) = if let Some(hash) = &args.commit_hash { - println!("Using commit hash: {}", hash.value()); - (Some("commit".to_string()), Some(hash.value())) - } else if let Some(tag) = &args.tag_name { - println!("Using tag: {}", tag.value()); - (Some("tag".to_string()), Some(tag.value())) - } else if let Some(ref branch) = args.branch_name { - println!("Using provided branch: {}", branch.value()); - (Some("branch".to_string()), Some(branch.value())) - } else { - // No specific git option provided - default branch case - println!("No specific git option provided, using flexible naming"); - (None, None) - }; + + // Replace the existing git option determination code with: + let (git_option_type, git_option_value) = + get_git_options(&args.commit_hash, &args.tag_name, &args.branch_name); // print the git option type and value if let (Some(opt_type), Some(opt_value)) = (&git_option_type, &git_option_value) { diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 2987a84..e26a378 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -468,3 +468,25 @@ pub fn get_or_create_snippets_dir() -> Result { Ok(snippets_dir) } + +/// Determines the git option type and value based on the provided arguments +pub fn get_git_options( + commit_hash: &Option, + tag_name: &Option, + branch_name: &Option, +) -> (Option, Option) { + if let Some(hash) = commit_hash { + println!("Using commit hash: {}", hash.value()); + (Some("commit".to_string()), Some(hash.value())) + } else if let Some(tag) = tag_name { + println!("Using tag: {}", tag.value()); + (Some("tag".to_string()), Some(tag.value())) + } else if let Some(branch) = branch_name { + println!("Using provided branch: {}", branch.value()); + (Some("branch".to_string()), Some(branch.value())) + } else { + // No specific git option provided - default branch case + println!("No specific git option provided, using flexible naming"); + (None, None) + } +} From 3f6c91ce56282bda1ccb44f180f036f5187a17f3 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 03:23:01 +0530 Subject: [PATCH 37/55] added missing imports --- macros/src/utils/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index e26a378..608c69d 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -6,8 +6,8 @@ use std::net::TcpStream; use std::path::{Path, PathBuf}; use std::time::Duration; use syn::visit::Visit; -use syn::Error; use syn::Result; +use syn::{Error, LitStr}; use crate::{caller_crate_root, source_excerpt, ItemVisitor}; From ae4daf0474422602ee330b1e080288f0a1027ce2 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 03:59:43 +0530 Subject: [PATCH 38/55] get_snippet_file_name and existing_snippet_path functions --- macros/src/lib.rs | 148 +-------------------------------------- macros/src/utils/mod.rs | 150 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 145 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 8a6939e..d49d1d3 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1286,101 +1286,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - // [git-url-hash]-[path-hash]-[ident_name]-[full-commit-hash].rs // first create the snippet file name based on the passed arguments - - let new_snippet = if let Some(hash) = &args.commit_hash { - // If commit hash is provided, use it regardless of internet connectivity - println!("Using provided commit hash: {}", hash.value()); - SnippetFile::new_with_commit( - git_url.value().as_str(), - "commit", - &hash.value(), - &args.file_path.value(), - &hash.value(), - &args.item_ident.as_ref().unwrap().to_string(), - ) - } else if let Some(tag) = &args.tag_name { - // Tag case - if has_internet { - let commit_sha = get_remote_commit_sha_without_clone( - git_url.value().as_str(), - None, - Some(tag.value().as_str()), - )?; - println!("commit hash for tag: {} -> {}", tag.value(), commit_sha); - SnippetFile::new_with_commit( - git_url.value().as_str(), - "tag", - &tag.value(), - &args.file_path.value(), - &commit_sha, - &args.item_ident.as_ref().unwrap().to_string(), - ) - } else { - println!("šŸ“” Offline mode: Creating snippet without commit hash for tag"); - SnippetFile::new_without_commit( - git_url.value().as_str(), - "tag", - &tag.value(), - &args.file_path.value(), - &args.item_ident.as_ref().unwrap().to_string(), - ) - } - } else if let Some(branch) = &args.branch_name { - // Explicit branch case - if has_internet { - let commit_sha = get_remote_commit_sha_without_clone( - git_url.value().as_str(), - Some(branch.value().as_str()), - None, - )?; - println!( - "commit hash for branch: {} -> {}", - branch.value(), - commit_sha - ); - SnippetFile::new_with_commit( - git_url.value().as_str(), - "branch", - branch.value().as_str(), - &args.file_path.value(), - &commit_sha, - &args.item_ident.as_ref().unwrap().to_string(), - ) - } else { - println!("šŸ“” Offline mode: Creating snippet without commit hash for branch"); - SnippetFile::new_without_commit( - git_url.value().as_str(), - "branch", - branch.value().as_str(), - &args.file_path.value(), - &args.item_ident.as_ref().unwrap().to_string(), - ) - } - } else { - // Default branch case - more flexible naming - if has_internet { - let commit_sha = - get_remote_commit_sha_without_clone(git_url.value().as_str(), None, None)?; - println!( - "branch not provided getting latest commit hash for the default branch -> {}", - commit_sha - ); - SnippetFile::new_for_default_branch( - git_url.value().as_str(), - &args.file_path.value(), - &args.item_ident.as_ref().unwrap().to_string(), - Some(&commit_sha), - ) - } else { - SnippetFile::new_for_default_branch( - git_url.value().as_str(), - &args.file_path.value(), - &args.item_ident.as_ref().unwrap().to_string(), - None, - ) - } - }; - + let new_snippet = get_snippet_file_name(git_url.value().as_str(), &args, has_internet)?; // snippet file name is created now check if it already exists println!( @@ -1393,55 +1299,8 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - // create the snippets directory if it doesn't exist let snippets_dir = get_or_create_snippets_dir()?; - // Check for existing snippet let existing_snippet_path = - if let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) { - if !has_internet { - println!( - "āœ… Found existing snippet (offline mode): .snippets/{}", - existing_snippet.full_name - ); - Some(existing_snippet.full_name) - } else { - // Online mode comparison - if let (Some(existing_hash), Some(new_hash)) = - (&existing_snippet.commit_hash, &new_snippet.commit_hash) - { - if existing_hash == new_hash { - println!( - "āœ… Existing snippet is up to date at: .snippets/{}", - existing_snippet.full_name - ); - Some(existing_snippet.full_name) - } else { - println!("ā„¹ļø Found existing snippet with different commit hash:"); - println!(" Current: {}", existing_hash); - println!(" New: {}", new_hash); - println!("šŸ”„ Removing outdated snippet..."); - - // Remove old snippet file - fs::remove_file(snippets_dir.join(&existing_snippet.full_name)) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to remove old snippet file: {}", e), - ) - })?; - None - } - } else { - None - } - } - } else if !has_internet { - println!("No matching snippet found and no internet connection available"); - return Err(Error::new( - Span::call_site(), - "No matching snippet found and no internet connection available", - )); - } else { - None - }; + check_existing_snippet(&new_snippet, has_internet, &snippets_dir)?; println!("existing_snippet_path: ----> {:?}", existing_snippet_path); @@ -1450,7 +1309,6 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - if let Some(snippet_name) = existing_snippet_path { println!("using existing snippet path skipping cloning"); let snippet_path = snippets_dir.join(snippet_name); - let content = fs::read_to_string(&snippet_path).map_err(|e| { Error::new( args.file_path.span(), @@ -1508,7 +1366,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ); let formatted_content = fix_indentation(&extracted_content); - let output = into_example(&formatted_content, lang); + let output: String = into_example(&formatted_content, lang); println!( "embed_internal_str ----> Final output length: {}", output.len() diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 608c69d..7092728 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -490,3 +490,153 @@ pub fn get_git_options( (None, None) } } + +/// Creates a new snippet file based on git options and internet connectivity +pub fn get_snippet_file_name( + git_url: &str, + args: &crate::EmbedArgs, + has_internet: bool, +) -> Result { + if let Some(hash) = &args.commit_hash { + // If commit hash is provided, use it regardless of internet connectivity + println!("Using provided commit hash: {}", hash.value()); + return Ok(SnippetFile::new_with_commit( + git_url, + "commit", + &hash.value(), + &args.file_path.value(), + &hash.value(), + &args.item_ident.as_ref().unwrap().to_string(), + )); + } + + if let Some(tag) = &args.tag_name { + return if has_internet { + let commit_sha = + get_remote_commit_sha_without_clone(git_url, None, Some(tag.value().as_str()))?; + println!("commit hash for tag: {} -> {}", tag.value(), commit_sha); + Ok(SnippetFile::new_with_commit( + git_url, + "tag", + &tag.value(), + &args.file_path.value(), + &commit_sha, + &args.item_ident.as_ref().unwrap().to_string(), + )) + } else { + println!("šŸ“” Offline mode: Creating snippet without commit hash for tag"); + Ok(SnippetFile::new_without_commit( + git_url, + "tag", + &tag.value(), + &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), + )) + }; + } + + if let Some(branch) = &args.branch_name { + return if has_internet { + let commit_sha = + get_remote_commit_sha_without_clone(git_url, Some(branch.value().as_str()), None)?; + println!( + "commit hash for branch: {} -> {}", + branch.value(), + commit_sha + ); + Ok(SnippetFile::new_with_commit( + git_url, + "branch", + branch.value().as_str(), + &args.file_path.value(), + &commit_sha, + &args.item_ident.as_ref().unwrap().to_string(), + )) + } else { + println!("šŸ“” Offline mode: Creating snippet without commit hash for branch"); + Ok(SnippetFile::new_without_commit( + git_url, + "branch", + branch.value().as_str(), + &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), + )) + }; + } + + // Default branch case - more flexible naming + if has_internet { + let commit_sha = get_remote_commit_sha_without_clone(git_url, None, None)?; + println!( + "branch not provided getting latest commit hash for the default branch -> {}", + commit_sha + ); + Ok(SnippetFile::new_for_default_branch( + git_url, + &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), + Some(&commit_sha), + )) + } else { + Ok(SnippetFile::new_for_default_branch( + git_url, + &args.file_path.value(), + &args.item_ident.as_ref().unwrap().to_string(), + None, + )) + } +} + +/// Checks if a snippet file already exists and handles updating if necessary +/// Returns Some(filename) if a valid snippet exists, None if we need to create a new one +pub fn check_existing_snippet( + new_snippet: &SnippetFile, + has_internet: bool, + snippets_dir: &Path, +) -> Result> { + let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) else { + if !has_internet { + return Err(Error::new( + Span::call_site(), + "No matching snippet found and no internet connection available", + )); + } + return Ok(None); + }; + + if !has_internet { + println!( + "āœ… Found existing snippet (offline mode): .snippets/{}", + existing_snippet.full_name + ); + return Ok(Some(existing_snippet.full_name)); + } + + // Online mode comparison + if let (Some(existing_hash), Some(new_hash)) = + (&existing_snippet.commit_hash, &new_snippet.commit_hash) + { + if existing_hash == new_hash { + println!( + "āœ… Existing snippet is up to date at: .snippets/{}", + existing_snippet.full_name + ); + return Ok(Some(existing_snippet.full_name)); + } + + println!("ā„¹ļø Found existing snippet with different commit hash:"); + println!(" Current: {}", existing_hash); + println!(" New: {}", new_hash); + println!("šŸ”„ Removing outdated snippet..."); + + // Remove old snippet file + fs::remove_file(snippets_dir.join(&existing_snippet.full_name)).map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to remove old snippet file: {}", e), + ) + })?; + } + + Ok(None) +} From eaf13207831636dc63aeafa2538a394c543c7a14 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 13:02:03 +0530 Subject: [PATCH 39/55] pub fn get_remote_commit_sha_without_clone( optimized function , to get refs of only the latest commit --- macros/src/utils/mod.rs | 132 ++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 58 deletions(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 7092728..2cdf56e 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -90,7 +90,11 @@ fn git_err_to_syn(err: git2::Error) -> syn::Error { syn::Error::new(Span::call_site(), format!("Git error: {}", err)) } -/// Gets commit SHA without cloning entire repo +/// Helper function to convert io::Error to syn::Error +fn io_err_to_syn(err: std::io::Error) -> syn::Error { + syn::Error::new(Span::call_site(), format!("IO error: {}", err)) +} + pub fn get_remote_commit_sha_without_clone( git_url: &str, branch: Option<&str>, @@ -100,79 +104,91 @@ pub fn get_remote_commit_sha_without_clone( .prefix("docify-temp-") .rand_bytes(5) .tempdir() - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to create temp dir: {}", e), - ) - })?; - println!("Created temp dir: {}", temp_dir.path().display()); + .map_err(io_err_to_syn)?; let repo = Repository::init(temp_dir.path()).map_err(git_err_to_syn)?; let mut remote = repo.remote_anonymous(git_url).map_err(git_err_to_syn)?; - println!("Initialized repo"); - - // First, fetch the remote HEAD to determine default branch - println!("ā„¹ļø Fetching remote references..."); - remote.connect(Direction::Fetch).map_err(git_err_to_syn)?; - - // Handle default branch resolution with proper error conversion - let default_branch = remote - .default_branch() - .map_err(git_err_to_syn)? - .as_str() - .map(String::from) - .ok_or_else(|| Error::new(Span::call_site(), "Invalid default branch name"))?; - - print!("defult branch --------> {}", default_branch); - remote.disconnect().map_err(git_err_to_syn)?; + // Set up fetch options + let mut fetch_opts = FetchOptions::new(); + fetch_opts.depth(1); // Only fetch the most recent commit - // Convert refs/heads/main to just main - let default_branch = default_branch - .strip_prefix("refs/heads/") - .unwrap_or(&default_branch); + let mut callbacks = RemoteCallbacks::new(); + callbacks.transfer_progress(|p| { + println!( + "Fetching: {}/{} objects", + p.received_objects(), + p.total_objects() + ); + true + }); + fetch_opts.remote_callbacks(callbacks); - println!("ā„¹ļø Default branch: {}", default_branch); + // First connect to get default branch if needed + remote.connect(Direction::Fetch).map_err(git_err_to_syn)?; - // Determine which refs to fetch - let refspecs = if let Some(tag_name) = tag { - vec![format!("refs/tags/{}:refs/tags/{}", tag_name, tag_name)] + // Determine which ref to fetch + let refspec = if let Some(tag_name) = tag { + format!("refs/tags/{}:refs/tags/{}", tag_name, tag_name) } else { - let branch_name = branch.unwrap_or(default_branch); - vec![format!( - "refs/heads/{}:refs/heads/{}", - branch_name, branch_name - )] + let branch_ref = if let Some(b) = branch { + format!("refs/heads/{}", b) + } else { + // Get default branch name + let default_branch = remote + .default_branch() + .map_err(git_err_to_syn)? + .as_str() + .map(String::from) + .ok_or_else(|| Error::new(Span::call_site(), "Invalid default branch name"))?; + + // Convert refs/heads/main to just refs/heads/main + if !default_branch.starts_with("refs/heads/") { + format!("refs/heads/{}", default_branch) + } else { + default_branch + } + }; + format!("{}:{}", branch_ref, branch_ref) }; - println!("Fetching refs: {:?}", refspecs); + println!("Fetching ref: {}", refspec); + + // Disconnect before fetch to ensure clean state + remote.disconnect().map_err(git_err_to_syn)?; + + // Fetch the specific ref remote - .fetch( - refspecs - .iter() - .map(|s| s.as_str()) - .collect::>() - .as_slice(), - None, - None, - ) + .fetch(&[&refspec], Some(&mut fetch_opts), None) .map_err(git_err_to_syn)?; - // Determine which commit to use - let commit_id = if let Some(tag_name) = tag { - let tag_ref = repo - .find_reference(&format!("refs/tags/{}", tag_name)) - .map_err(git_err_to_syn)?; - tag_ref.peel_to_commit().map_err(git_err_to_syn)?.id() + // Get commit ID + let reference = if let Some(tag_name) = tag { + repo.find_reference(&format!("refs/tags/{}", tag_name)) + .map_err(git_err_to_syn)? } else { - let branch_name = branch.unwrap_or(default_branch); - let reference = repo - .find_reference(&format!("refs/heads/{}", branch_name)) - .map_err(git_err_to_syn)?; - reference.peel_to_commit().map_err(git_err_to_syn)?.id() + let ref_name = if let Some(b) = branch { + format!("refs/heads/{}", b) + } else { + // Use the actual fetched ref name + let default_branch = remote + .default_branch() + .map_err(git_err_to_syn)? + .as_str() + .map(String::from) + .ok_or_else(|| Error::new(Span::call_site(), "Invalid default branch name"))?; + + if !default_branch.starts_with("refs/heads/") { + format!("refs/heads/{}", default_branch) + } else { + default_branch + } + }; + repo.find_reference(&ref_name).map_err(git_err_to_syn)? }; + let commit_id = reference.peel_to_commit().map_err(git_err_to_syn)?.id(); + Ok(commit_id.to_string()) } From cf4849d0b0b9ff1fd8953a8c88f0319517f1e6da Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 17:19:09 +0530 Subject: [PATCH 40/55] commented out git refspec cloning progress --- macros/src/utils/mod.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 2cdf56e..bfad3b5 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -113,16 +113,18 @@ pub fn get_remote_commit_sha_without_clone( let mut fetch_opts = FetchOptions::new(); fetch_opts.depth(1); // Only fetch the most recent commit - let mut callbacks = RemoteCallbacks::new(); - callbacks.transfer_progress(|p| { - println!( - "Fetching: {}/{} objects", - p.received_objects(), - p.total_objects() - ); - true - }); - fetch_opts.remote_callbacks(callbacks); + println!("fetching ref"); + // let mut callbacks = RemoteCallbacks::new(); + // callbacks.transfer_progress(|p| { + // println!( + // "Fetching: {}/{} objects", + // p.received_objects(), + // p.total_objects() + // ); + // true + // }); + + // fetch_opts.remote_callbacks(callbacks); // First connect to get default branch if needed remote.connect(Direction::Fetch).map_err(git_err_to_syn)?; From 7363483f55b7989452ae426225ee0d290a15dbc8 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 17:27:45 +0530 Subject: [PATCH 41/55] stripout .git from git_url it it's present --- macros/src/utils/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index bfad3b5..366dc09 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -459,11 +459,15 @@ fn hash_git_option(option_type: &str, value: &str) -> String { hash_string(&format!("{}-{}", option_type, value)) } -/// Helper function to get default branch +/// Normalizes a git URL by removing trailing .git if present +fn normalize_git_url(url: &str) -> &str { + url.strip_suffix(".git").unwrap_or(url) +} fn hash_git_url(url: &str) -> String { println!("ā„¹ļø Hashing git URL: {}", url); - hash_string(url) + let normalized_url = normalize_git_url(url); + hash_string(normalized_url) } /// Creates and returns the snippets directory path, ensuring it exists From 5a41628f16e38dc72d68422a3f1fe85e07635f4e Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 17:51:08 +0530 Subject: [PATCH 42/55] add check for empty string in file path in positional args --- macros/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index d49d1d3..774b7bd 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -563,7 +563,7 @@ struct EmbedArgs { item_ident: Option, } -// Manually implement Debug +// implementing Debug for EmbedArgs for easier debugging impl std::fmt::Debug for EmbedArgs { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("EmbedArgs") @@ -652,11 +652,15 @@ impl EmbedArgs { } Ok(()) } - /// Parses positional arguments format fn parse_positional(input: ParseStream) -> Result { let file_path: LitStr = input.parse()?; + // Check for empty file path + if file_path.value().trim().is_empty() { + return Err(Error::new(file_path.span(), "File path cannot be empty")); + } + let item_ident = if input.peek(Token![,]) { input.parse::()?; Some(input.parse()?) From 92b994f3474a31834d73ff725e92db92f97dee14 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sun, 3 Nov 2024 23:52:53 +0530 Subject: [PATCH 43/55] changed has_internet to allow_updates , which will be false if DOCS_RS=1 or DOCIFY_DISABLE_UPDATES=1 --- macros/src/lib.rs | 28 ++++++++++++++++++++++------ macros/src/utils/mod.rs | 14 +++++++------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 774b7bd..3c4e7cc 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1261,14 +1261,28 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - // get the root of the crate let crate_root = caller_crate_root() .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; - if let Some(git_url) = &args.git_url { - let has_internet = check_internet_connectivity(); + let allow_updates = + !std::env::var("DOCS_RS").is_ok() && !std::env::var("DOCIFY_DISABLE_UPDATES").is_ok(); + + println!("env DOCS_RS: {:?}", std::env::var("DOCS_RS")); println!( - "\nšŸŒ Internet connectivity: {}", - if has_internet { "Online" } else { "Offline" } + "env DOCIFY_DISABLE_UPDATES: {:?}", + std::env::var("DOCIFY_DISABLE_UPDATES") ); + println!( + "\nšŸŒ Updates {}", + if allow_updates { + "allowing updates" + } else if std::env::var("DOCS_RS").is_ok() { + "Offline (docs.rs build)" + } else if std::env::var("DOCIFY_DISABLE_UPDATES").is_ok() { + "Offline (updates disabled)" + } else { + "updates disabled" + } + ); // Determine git option type and value // Replace the existing git option determination code with: @@ -1290,7 +1304,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - // [git-url-hash]-[path-hash]-[ident_name]-[full-commit-hash].rs // first create the snippet file name based on the passed arguments - let new_snippet = get_snippet_file_name(git_url.value().as_str(), &args, has_internet)?; + let new_snippet = get_snippet_file_name(git_url.value().as_str(), &args, allow_updates)?; // snippet file name is created now check if it already exists println!( @@ -1304,7 +1318,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let snippets_dir = get_or_create_snippets_dir()?; let existing_snippet_path = - check_existing_snippet(&new_snippet, has_internet, &snippets_dir)?; + check_existing_snippet(&new_snippet, allow_updates, &snippets_dir)?; println!("existing_snippet_path: ----> {:?}", existing_snippet_path); @@ -1323,6 +1337,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ), ) })?; + let formatted_content = fix_indentation(&content); let output = into_example(&formatted_content, lang); println!( @@ -1338,6 +1353,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - git_url.value().as_str(), &new_snippet.commit_hash.as_ref().unwrap(), )?; + println!("āœ… Cloned and checked out repo"); let source_path = repo_dir.join(&args.file_path.value()); diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index 366dc09..cbeb798 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -517,7 +517,7 @@ pub fn get_git_options( pub fn get_snippet_file_name( git_url: &str, args: &crate::EmbedArgs, - has_internet: bool, + allow_updates: bool, ) -> Result { if let Some(hash) = &args.commit_hash { // If commit hash is provided, use it regardless of internet connectivity @@ -533,7 +533,7 @@ pub fn get_snippet_file_name( } if let Some(tag) = &args.tag_name { - return if has_internet { + return if allow_updates { let commit_sha = get_remote_commit_sha_without_clone(git_url, None, Some(tag.value().as_str()))?; println!("commit hash for tag: {} -> {}", tag.value(), commit_sha); @@ -558,7 +558,7 @@ pub fn get_snippet_file_name( } if let Some(branch) = &args.branch_name { - return if has_internet { + return if allow_updates { let commit_sha = get_remote_commit_sha_without_clone(git_url, Some(branch.value().as_str()), None)?; println!( @@ -587,7 +587,7 @@ pub fn get_snippet_file_name( } // Default branch case - more flexible naming - if has_internet { + if allow_updates { let commit_sha = get_remote_commit_sha_without_clone(git_url, None, None)?; println!( "branch not provided getting latest commit hash for the default branch -> {}", @@ -613,11 +613,11 @@ pub fn get_snippet_file_name( /// Returns Some(filename) if a valid snippet exists, None if we need to create a new one pub fn check_existing_snippet( new_snippet: &SnippetFile, - has_internet: bool, + allow_updates: bool, snippets_dir: &Path, ) -> Result> { let Some(existing_snippet) = SnippetFile::find_existing(&new_snippet.prefix) else { - if !has_internet { + if !allow_updates { return Err(Error::new( Span::call_site(), "No matching snippet found and no internet connection available", @@ -626,7 +626,7 @@ pub fn check_existing_snippet( return Ok(None); }; - if !has_internet { + if !allow_updates { println!( "āœ… Found existing snippet (offline mode): .snippets/{}", existing_snippet.full_name From cce3497613e279e8fdd7a1ddadb7c4e5fc9611a6 Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 13:04:35 +0530 Subject: [PATCH 44/55] added test 1 embed invalid args test failing --- macros/src/tests.rs | 122 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/macros/src/tests.rs b/macros/src/tests.rs index cd926b2..ba26356 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -1,4 +1,5 @@ use super::*; +use proc_macro2::TokenStream as TokenStream2; use std::fs; use std::path::PathBuf; use tempfile::TempDir; @@ -196,3 +197,124 @@ fn bar() { "#; assert_eq!(fix_leading_indentation(input), output); } + +#[test] +fn test_embed_args_basic() { + // Test basic file path only + let args = parse2::(quote!("src/lib.rs")).unwrap(); + assert!(args.git_url.is_none()); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert!(args.item_ident.is_none()); + assert!(args.branch_name.is_none()); + assert!(args.commit_hash.is_none()); + assert!(args.tag_name.is_none()); + + // Test file path with item + let args = parse2::(quote!("src/lib.rs", my_function)).unwrap(); + assert!(args.git_url.is_none()); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); +} + +#[test] +fn test_embed_args_git() { + // Test with git URL + let args = parse2::(quote!(git: "https://github.com/user/repo", path: "src/lib.rs")) + .unwrap(); + assert_eq!( + args.git_url.unwrap().value(), + "https://github.com/user/repo" + ); + assert_eq!(args.file_path.value(), "src/lib.rs"); + + // Test with git URL and branch + let args = parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main" + )) + .unwrap(); + assert_eq!(args.branch_name.unwrap().value(), "main"); +} + +#[test] +fn test_embed_args_git_refs() { + // Test with commit hash + let args = parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + commit: "abc123" + )) + .unwrap(); + assert_eq!(args.commit_hash.unwrap().value(), "abc123"); + + // Test with tag + let args = parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + tag: "v1.0.0" + )) + .unwrap(); + assert_eq!(args.tag_name.unwrap().value(), "v1.0.0"); +} + +#[test] +fn test_embed_args_with_item() { + // Test git URL with item + let args = parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + item: my_function + )) + .unwrap(); + assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); +} + +#[test] +fn test_embed_args_invalid() { + // Test empty path + assert!(parse2::(quote!("")).is_err()); + + // Test multiple git refs (should fail) + assert!(parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main", + tag: "v1.0.0" + )) + .is_err()); + + // Test git refs without git URL (should fail) + assert!(parse2::(quote!( + path: "src/lib.rs", + branch: "main" + )) + .is_err()); + + // Test invalid path with git URL + assert!(parse2::(quote!( + git: "https://github.com/user/repo", + path: "../src/lib.rs" + )) + .is_err()); +} + +#[test] +fn test_embed_args_complex() { + // Test full featured usage + let args = parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "feature/new", + item: test_function + )) + .unwrap(); + + assert_eq!( + args.git_url.unwrap().value(), + "https://github.com/user/repo" + ); + assert_eq!(args.file_path.value(), "src/lib.rs"); + assert_eq!(args.branch_name.unwrap().value(), "feature/new"); + assert_eq!(args.item_ident.unwrap().to_string(), "test_function"); +} From 845da66f0d96b213e0347457545ddae146460ccf Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 13:13:53 +0530 Subject: [PATCH 45/55] updated tests --- macros/src/tests.rs | 66 +++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/macros/src/tests.rs b/macros/src/tests.rs index ba26356..d42b428 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -269,36 +269,62 @@ fn test_embed_args_with_item() { .unwrap(); assert_eq!(args.item_ident.unwrap().to_string(), "my_function"); } - #[test] fn test_embed_args_invalid() { // Test empty path assert!(parse2::(quote!("")).is_err()); // Test multiple git refs (should fail) - assert!(parse2::(quote!( - git: "https://github.com/user/repo", - path: "src/lib.rs", - branch: "main", - tag: "v1.0.0" - )) - .is_err()); + assert!( + parse2::(quote!( + git: "https://github.com/user/repo", + path: "src/lib.rs", + branch: "main", + tag: "v1.0.0" + )) + .is_err(), + "Should fail when multiple git refs are provided" + ); // Test git refs without git URL (should fail) - assert!(parse2::(quote!( - path: "src/lib.rs", - branch: "main" - )) - .is_err()); + assert!( + parse2::(quote!( + path: "src/lib.rs", + branch: "main" + )) + .is_err(), + "Should fail when git refs are provided without git URL" + ); - // Test invalid path with git URL - assert!(parse2::(quote!( - git: "https://github.com/user/repo", - path: "../src/lib.rs" - )) - .is_err()); -} + // Test missing path with git URL (should fail) + assert!( + parse2::(quote!( + git: "https://github.com/user/repo" + )) + .is_err(), + "Should fail when path is missing" + ); + + // Test invalid URL format + assert!( + parse2::(quote!( + git: "not a valid url", + path: "src/lib.rs" + )) + .is_err(), + "Should fail with invalid git URL format" + ); + // Test empty git URL + assert!( + parse2::(quote!( + git: "", + path: "src/lib.rs" + )) + .is_err(), + "Should fail with empty git URL" + ); +} #[test] fn test_embed_args_complex() { // Test full featured usage From 70cc180b17518074119e2e38d0ebd5c20a90d603 Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 23:01:15 +0530 Subject: [PATCH 46/55] validate git url when passed as named args --- macros/src/lib.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 3c4e7cc..b95092d 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -607,6 +607,7 @@ impl EmbedArgs { self.validate_file_path()?; self.validate_git_refs()?; self.validate_git_dependencies()?; + self.validate_git_url()?; Ok(()) } @@ -624,6 +625,20 @@ impl EmbedArgs { Ok(()) } + /// Ensures git URL is valid if provided + fn validate_git_url(&self) -> Result<()> { + if let Some(git_url) = &self.git_url { + let url = git_url.value(); + if !url.starts_with("https://") { + return Err(Error::new( + git_url.span(), + "Please provide a valid Git URL starting with 'https://'", + )); + } + } + Ok(()) + } + /// Ensures only one git reference type is specified fn validate_git_refs(&self) -> Result<()> { let ref_count = [&self.branch_name, &self.commit_hash, &self.tag_name] From 9f2b7ac7f82ec1ad5079d0b6bf8ce30afba10f3a Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 23:40:39 +0530 Subject: [PATCH 47/55] moved old file path validation checks into struct impl --- macros/src/lib.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index b95092d..a0bc0b2 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -613,15 +613,26 @@ impl EmbedArgs { /// Ensures file path is valid based on context fn validate_file_path(&self) -> Result<()> { - if self.git_url.is_none() - && (self.file_path.value().starts_with("http://") - || self.file_path.value().starts_with("https://")) - { + let path = self.file_path.value(); + + // Check for URLs in file path when git_url is not provided + if self.git_url.is_none() && (path.starts_with("http://") || path.starts_with("https://")) { return Err(Error::new( self.file_path.span(), "File path cannot be a URL. Use git: \"url\" for git repositories", )); } + + // Check for paths starting with ".." or "/" + if path.starts_with("..") || path.starts_with("/") { + let error_msg = if self.git_url.is_some() { + "When using git_url, please provide the correct file path in your git source. The path should not start with '..' or '/'." + } else { + "You can only embed files which are present in the current crate. For any other files, please provide the git_url to embed." + }; + return Err(Error::new(self.file_path.span(), error_msg)); + } + Ok(()) } @@ -1246,12 +1257,6 @@ fn source_excerpt<'a, T: ToTokens>( fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) -> Result { let args: EmbedArgs = parse2::(tokens.into())?; - println!( - "embed_internal_str ----> args: git_url: {:?}, file_path: {}, item_ident: {:?}", - args.git_url.as_ref().map(|url| url.value()), - args.file_path.value(), - args.item_ident.as_ref().map(|i| i.to_string()) - ); // Check if the file_path starts with "../" and git_url is not provided if (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("/")) From 3847fd57f6c2627a257af10d2cd81c5d085dbd0a Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 23:45:09 +0530 Subject: [PATCH 48/55] used pattern matching in set_arg --- macros/src/lib.rs | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index a0bc0b2..1c0d7f7 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -728,22 +728,17 @@ impl NamedArgsBuilder { fn set_arg(&mut self, input: ParseStream) -> Result<()> { let lookahead = input.lookahead1(); - if lookahead.peek(kw::git) { - self.parse_git_url(input)?; - } else if lookahead.peek(kw::path) { - self.parse_file_path(input)?; - } else if lookahead.peek(kw::branch) { - self.parse_branch(input)?; - } else if lookahead.peek(kw::commit) { - self.parse_commit(input)?; - } else if lookahead.peek(kw::tag) { - self.parse_tag(input)?; - } else if lookahead.peek(kw::item) { - self.parse_item(input)?; - } else { - return Err(lookahead.error()); - } - + match () { + _ if lookahead.peek(kw::git) => self.parse_git_url(input), + _ if lookahead.peek(kw::path) => self.parse_file_path(input), + _ if lookahead.peek(kw::branch) => self.parse_branch(input), + _ if lookahead.peek(kw::commit) => self.parse_commit(input), + _ if lookahead.peek(kw::tag) => self.parse_tag(input), + _ if lookahead.peek(kw::item) => self.parse_item(input), + _ => return Err(lookahead.error()), + }?; + + // Handle trailing comma if more input exists if !input.is_empty() { input.parse::()?; } From f7bf8992ff236ab87d32bef52279fc73676d1528 Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 23:49:41 +0530 Subject: [PATCH 49/55] removed the extra check --- macros/src/lib.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 1c0d7f7..dbc2f59 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1253,26 +1253,6 @@ fn source_excerpt<'a, T: ToTokens>( fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) -> Result { let args: EmbedArgs = parse2::(tokens.into())?; - // Check if the file_path starts with "../" and git_url is not provided - if (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("/")) - && args.git_url.is_none() - { - return Err(Error::new( - args.file_path.span(), - "You can only embed files which are present in the current crate. For any other files, please provide the git_url to embed." - )); - } - - // Check if git_url is provided and path starts with ".." or "../" - if args.git_url.is_some() - && (args.file_path.value().starts_with("..") || args.file_path.value().starts_with("/")) - { - return Err(Error::new( - args.file_path.span(), - "When using git_url, please provide the correct file path in your git source. The path should not start with '..' or '../'." - )); - } - // get the root of the crate let crate_root = caller_crate_root() .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; From 97cd4014dd3cdbc7737d3edcdaf2711a6c9b44f1 Mon Sep 17 00:00:00 2001 From: Prakash Date: Mon, 4 Nov 2024 23:56:37 +0530 Subject: [PATCH 50/55] removed print statements from embed_internal_str --- macros/src/lib.rs | 80 +++-------------------------------------------- 1 file changed, 5 insertions(+), 75 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index dbc2f59..a6fc636 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1260,36 +1260,10 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let allow_updates = !std::env::var("DOCS_RS").is_ok() && !std::env::var("DOCIFY_DISABLE_UPDATES").is_ok(); - println!("env DOCS_RS: {:?}", std::env::var("DOCS_RS")); - println!( - "env DOCIFY_DISABLE_UPDATES: {:?}", - std::env::var("DOCIFY_DISABLE_UPDATES") - ); - - println!( - "\nšŸŒ Updates {}", - if allow_updates { - "allowing updates" - } else if std::env::var("DOCS_RS").is_ok() { - "Offline (docs.rs build)" - } else if std::env::var("DOCIFY_DISABLE_UPDATES").is_ok() { - "Offline (updates disabled)" - } else { - "updates disabled" - } - ); // Determine git option type and value - // Replace the existing git option determination code with: - let (git_option_type, git_option_value) = - get_git_options(&args.commit_hash, &args.tag_name, &args.branch_name); - - // print the git option type and value - if let (Some(opt_type), Some(opt_value)) = (&git_option_type, &git_option_value) { - println!("Git option type: {}, value: {}", opt_type, opt_value); - } else { - println!("Using flexible naming without git options"); - } + // git option type and value is determined in get_snippet_file_name + // git option type value is calcualted based on , branch / commit / tag and is hashed into 8 characters // if git option it present the name of the snippet will be of the format // [git-url-hash]-[git-option-hash]-[path-hash]-[ident_name]-[full-commit-hash].rs @@ -1302,25 +1276,15 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let new_snippet = get_snippet_file_name(git_url.value().as_str(), &args, allow_updates)?; // snippet file name is created now check if it already exists - println!( - "created snippet file name based on the passed arguments ----> {}", - new_snippet.full_name - ); - - println!("\nšŸ” Now Checking if the snippet file already exists"); - // create the snippets directory if it doesn't exist let snippets_dir = get_or_create_snippets_dir()?; let existing_snippet_path = check_existing_snippet(&new_snippet, allow_updates, &snippets_dir)?; - println!("existing_snippet_path: ----> {:?}", existing_snippet_path); - // Use existing snippet if available, otherwise proceed with cloning if let Some(snippet_name) = existing_snippet_path { - println!("using existing snippet path skipping cloning"); let snippet_path = snippets_dir.join(snippet_name); let content = fs::read_to_string(&snippet_path).map_err(|e| { Error::new( @@ -1335,10 +1299,6 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - let formatted_content = fix_indentation(&content); let output = into_example(&formatted_content, lang); - println!( - "embed_internal_str ----> Final output length: {}", - output.len() - ); return Ok(output); } @@ -1349,25 +1309,15 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - &new_snippet.commit_hash.as_ref().unwrap(), )?; - println!("āœ… Cloned and checked out repo"); - let source_path = repo_dir.join(&args.file_path.value()); - println!("Reading source file from: {}", source_path.display()); + // extract the item from the file let extracted_content: String = extract_item_from_file(&source_path, &args.item_ident.as_ref().unwrap().to_string())?; - println!( - "extracted_content: from extract_item_from_file using ident ----> {} \n\n extracted content {}", - args.item_ident.as_ref().unwrap().to_string(), - extracted_content - ); - let snippet_path = snippets_dir.join(&new_snippet.full_name); - println!( - "Writing content to snippet file: {}", - snippet_path.display() - ); + + // write the extracted content to the snippet file fs::write(&snippet_path, extracted_content.clone()).map_err(|e| { Error::new( Span::call_site(), @@ -1375,23 +1325,12 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - ) })?; - println!( - "āœ… Wrote content to snippet file at path: {}", - snippet_path.display() - ); - let formatted_content = fix_indentation(&extracted_content); let output: String = into_example(&formatted_content, lang); - println!( - "embed_internal_str ----> Final output length: {}", - output.len() - ); Ok(output) } else { - println!("Detected local file embedding"); let file_path = crate_root.join(args.file_path.value()); - println!("embed_internal_str ----> File path: {:?}", file_path); let source_code = fs::read_to_string(&file_path).map_err(|e| { Error::new( @@ -1405,10 +1344,8 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - })?; let source_file = syn::parse_file(&source_code)?; - println!("embed_internal_str ----> Parsed source file successfully"); let output = if let Some(ident) = args.item_ident.as_ref() { - println!("embed_internal_str ----> Searching for item: {}", ident); let mut visitor = ItemVisitor { search: ident.clone(), results: Vec::new(), @@ -1435,16 +1372,9 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - } results.join("\n") } else { - println!("embed_internal_str ----> No specific item requested, using entire source"); into_example(source_code.as_str(), lang) }; - println!("Successfully embedded local content"); - println!( - "embed_internal_str ----> Final output length: {}", - output.len() - ); - Ok(output) } } From 2de26c0ee550b79d50cf9ae4df1c7f939fb20759 Mon Sep 17 00:00:00 2001 From: Prakash Date: Tue, 5 Nov 2024 00:18:48 +0530 Subject: [PATCH 51/55] removed all print statements from mod.rs --- macros/src/utils/mod.rs | 171 +--------------------------------------- 1 file changed, 2 insertions(+), 169 deletions(-) diff --git a/macros/src/utils/mod.rs b/macros/src/utils/mod.rs index cbeb798..a85235d 100644 --- a/macros/src/utils/mod.rs +++ b/macros/src/utils/mod.rs @@ -2,22 +2,14 @@ use git2::{Direction, FetchOptions, RemoteCallbacks, Repository}; use proc_macro2::Span; use sha2::{Digest, Sha256}; use std::fs; -use std::net::TcpStream; use std::path::{Path, PathBuf}; -use std::time::Duration; use syn::visit::Visit; +use syn::Error; use syn::Result; -use syn::{Error, LitStr}; use crate::{caller_crate_root, source_excerpt, ItemVisitor}; pub fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result { - println!( - "inside extract_item_from_file ----> Extracting item '{}' from '{}'", - item_ident, - file_path.display() - ); - let source_code = fs::read_to_string(file_path).map_err(|e| { Error::new( Span::call_site(), @@ -29,20 +21,11 @@ pub fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result Source code: {}", - source_code - ); - let mut visitor = ItemVisitor { search: syn::parse_str(item_ident)?, results: Vec::new(), }; visitor.visit_file(&syn::parse_file(&source_code)?); - println!( - "inside extract_item_from_file ----> Visitor results: {:?}", - visitor.results - ); if visitor.results.is_empty() { return Err(Error::new( Span::call_site(), @@ -54,36 +37,11 @@ pub fn extract_item_from_file(file_path: &Path, item_ident: &str) -> Result bool { - // List of reliable hosts and ports to try - let hosts = [ - ("8.8.8.8", 53), // Google DNS - ("1.1.1.1", 53), // Cloudflare DNS - ("208.67.222.222", 53), // OpenDNS - ]; - - // Set a timeout for connection attempts - let timeout = Duration::from_secs(1); - - for &(host, port) in hosts.iter() { - if let Ok(stream) = TcpStream::connect((host, port)) { - // Set the timeout for read/write operations - if stream.set_read_timeout(Some(timeout)).is_ok() - && stream.set_write_timeout(Some(timeout)).is_ok() - { - return true; - } - } - } - - false -} // comment /// Helper function to convert git2::Error to syn::Error fn git_err_to_syn(err: git2::Error) -> syn::Error { @@ -113,19 +71,6 @@ pub fn get_remote_commit_sha_without_clone( let mut fetch_opts = FetchOptions::new(); fetch_opts.depth(1); // Only fetch the most recent commit - println!("fetching ref"); - // let mut callbacks = RemoteCallbacks::new(); - // callbacks.transfer_progress(|p| { - // println!( - // "Fetching: {}/{} objects", - // p.received_objects(), - // p.total_objects() - // ); - // true - // }); - - // fetch_opts.remote_callbacks(callbacks); - // First connect to get default branch if needed remote.connect(Direction::Fetch).map_err(git_err_to_syn)?; @@ -154,8 +99,6 @@ pub fn get_remote_commit_sha_without_clone( format!("{}:{}", branch_ref, branch_ref) }; - println!("Fetching ref: {}", refspec); - // Disconnect before fetch to ensure clean state remote.disconnect().map_err(git_err_to_syn)?; @@ -209,10 +152,6 @@ pub fn get_or_create_commit_dir(git_url: &str, commit_sha: &str) -> Result {}", - repo_name - ); // Use first 8 chars of commit hash let short_commit = &commit_sha[..8]; @@ -222,10 +161,8 @@ pub fn get_or_create_commit_dir(git_url: &str, commit_sha: &str) -> Result Result: {} returning existing dir path ----> {}", - commit_sha, - commit_dir.display() - ); return Ok(commit_dir); } - println!("Cloning new repo for commit: {}", commit_sha); - let mut callbacks = RemoteCallbacks::new(); - callbacks.transfer_progress(|p| { - println!( - "Fetching: {}/{} objects", - p.received_objects(), - p.total_objects() - ); - true - }); + callbacks.transfer_progress(|_p| true); let mut fetch_opts = FetchOptions::new(); fetch_opts.remote_callbacks(callbacks); @@ -304,10 +227,6 @@ impl SnippetFile { path: &str, item_ident: &str, ) -> Self { - println!("\nšŸ“ Creating new SnippetFile..."); - println!("ā„¹ļø Input path: {}", path); - println!("ā„¹ļø Item identifier: {}", item_ident); - let prefix = format!( "{}-{}-{}-{}", hash_git_url(git_url), @@ -315,8 +234,6 @@ impl SnippetFile { hash_string(path), item_ident, ); - println!("ā„¹ļø Generated prefix: {}", prefix); - Self { prefix: prefix.clone(), commit_hash: None, @@ -332,10 +249,6 @@ impl SnippetFile { commit_sha: &str, item_ident: &str, ) -> Self { - println!("\nšŸ“ Creating new SnippetFile..."); - println!("ā„¹ļø Input path: {}", path); - println!("ā„¹ļø Item identifier: {}", item_ident); - let prefix = format!( "{}-{}-{}-{}", hash_git_url(git_url), @@ -343,11 +256,8 @@ impl SnippetFile { hash_string(path), item_ident, ); - println!("ā„¹ļø Generated prefix: {}", prefix); let full_name = format!("{}-{}.rs", prefix, commit_sha); - println!("āœ… Created snippet filename: {}", full_name); - Self { prefix, commit_hash: Some(commit_sha.to_string()), @@ -356,49 +266,33 @@ impl SnippetFile { } pub fn find_existing(prefix: &str) -> Option { - println!("\nšŸ” Looking for existing snippet with prefix: {}", prefix); - // Get the crate root path let crate_root = match crate::caller_crate_root() { Some(root) => root, None => { - println!("āŒ Failed to resolve crate root"); return None; } }; // Use absolute path by joining with crate root let snippets_dir = crate_root.join(".snippets"); - println!("šŸ“ Checking snippets directory: {}", snippets_dir.display()); // Check if directory exists and is actually a directory if !snippets_dir.exists() { - println!( - "āŒ .snippets directory does not exist at {}", - snippets_dir.display() - ); return None; } fs::read_dir(snippets_dir).ok()?.find_map(|entry| { let entry = entry.ok()?; - println!("entry: {:?}", entry); let file_name = entry.file_name().to_string_lossy().to_string(); - println!("ā„¹ļø Checking file: {}", file_name); - if file_name.starts_with(prefix) { - println!("āœ… Found matching file!"); // Extract commit hash from filename if it exists let commit_hash = file_name .strip_suffix(".rs")? .rsplit('-') .next() .map(|s| s.to_string()); - println!( - "ā„¹ļø Extracted commit hash from existing file: {:?}", - commit_hash - ); Some(Self { prefix: prefix.to_string(), @@ -418,28 +312,21 @@ impl SnippetFile { item_ident: &str, commit_sha: Option<&str>, ) -> Self { - println!("\nšŸ“ Creating new SnippetFile for default branch case"); - println!("ā„¹ļø Input path: {}", path); - println!("ā„¹ļø Item identifier: {}", item_ident); - let prefix = format!( "{}-{}-{}", hash_git_url(git_url), hash_string(path), item_ident, ); - println!("ā„¹ļø Generated prefix: {}", prefix); if let Some(commit) = commit_sha { let full_name = format!("{}-{}.rs", prefix, commit); - println!("āœ… Created snippet filename with commit: {}", full_name); Self { prefix, commit_hash: Some(commit.to_string()), full_name, } } else { - println!("āœ… Created snippet filename without commit"); Self { prefix: prefix.clone(), commit_hash: None, @@ -465,7 +352,6 @@ fn normalize_git_url(url: &str) -> &str { } fn hash_git_url(url: &str) -> String { - println!("ā„¹ļø Hashing git URL: {}", url); let normalized_url = normalize_git_url(url); hash_string(normalized_url) } @@ -476,11 +362,6 @@ pub fn get_or_create_snippets_dir() -> Result { .ok_or_else(|| Error::new(Span::call_site(), "Failed to resolve caller crate root"))?; let snippets_dir = crate_root.join(".snippets"); - println!( - "šŸ“ Ensuring snippets directory exists at: {}", - snippets_dir.display() - ); - fs::create_dir_all(&snippets_dir).map_err(|e| { Error::new( Span::call_site(), @@ -491,28 +372,6 @@ pub fn get_or_create_snippets_dir() -> Result { Ok(snippets_dir) } -/// Determines the git option type and value based on the provided arguments -pub fn get_git_options( - commit_hash: &Option, - tag_name: &Option, - branch_name: &Option, -) -> (Option, Option) { - if let Some(hash) = commit_hash { - println!("Using commit hash: {}", hash.value()); - (Some("commit".to_string()), Some(hash.value())) - } else if let Some(tag) = tag_name { - println!("Using tag: {}", tag.value()); - (Some("tag".to_string()), Some(tag.value())) - } else if let Some(branch) = branch_name { - println!("Using provided branch: {}", branch.value()); - (Some("branch".to_string()), Some(branch.value())) - } else { - // No specific git option provided - default branch case - println!("No specific git option provided, using flexible naming"); - (None, None) - } -} - /// Creates a new snippet file based on git options and internet connectivity pub fn get_snippet_file_name( git_url: &str, @@ -521,7 +380,6 @@ pub fn get_snippet_file_name( ) -> Result { if let Some(hash) = &args.commit_hash { // If commit hash is provided, use it regardless of internet connectivity - println!("Using provided commit hash: {}", hash.value()); return Ok(SnippetFile::new_with_commit( git_url, "commit", @@ -536,7 +394,6 @@ pub fn get_snippet_file_name( return if allow_updates { let commit_sha = get_remote_commit_sha_without_clone(git_url, None, Some(tag.value().as_str()))?; - println!("commit hash for tag: {} -> {}", tag.value(), commit_sha); Ok(SnippetFile::new_with_commit( git_url, "tag", @@ -546,7 +403,6 @@ pub fn get_snippet_file_name( &args.item_ident.as_ref().unwrap().to_string(), )) } else { - println!("šŸ“” Offline mode: Creating snippet without commit hash for tag"); Ok(SnippetFile::new_without_commit( git_url, "tag", @@ -561,11 +417,6 @@ pub fn get_snippet_file_name( return if allow_updates { let commit_sha = get_remote_commit_sha_without_clone(git_url, Some(branch.value().as_str()), None)?; - println!( - "commit hash for branch: {} -> {}", - branch.value(), - commit_sha - ); Ok(SnippetFile::new_with_commit( git_url, "branch", @@ -575,7 +426,6 @@ pub fn get_snippet_file_name( &args.item_ident.as_ref().unwrap().to_string(), )) } else { - println!("šŸ“” Offline mode: Creating snippet without commit hash for branch"); Ok(SnippetFile::new_without_commit( git_url, "branch", @@ -589,10 +439,6 @@ pub fn get_snippet_file_name( // Default branch case - more flexible naming if allow_updates { let commit_sha = get_remote_commit_sha_without_clone(git_url, None, None)?; - println!( - "branch not provided getting latest commit hash for the default branch -> {}", - commit_sha - ); Ok(SnippetFile::new_for_default_branch( git_url, &args.file_path.value(), @@ -627,10 +473,6 @@ pub fn check_existing_snippet( }; if !allow_updates { - println!( - "āœ… Found existing snippet (offline mode): .snippets/{}", - existing_snippet.full_name - ); return Ok(Some(existing_snippet.full_name)); } @@ -639,18 +481,9 @@ pub fn check_existing_snippet( (&existing_snippet.commit_hash, &new_snippet.commit_hash) { if existing_hash == new_hash { - println!( - "āœ… Existing snippet is up to date at: .snippets/{}", - existing_snippet.full_name - ); return Ok(Some(existing_snippet.full_name)); } - println!("ā„¹ļø Found existing snippet with different commit hash:"); - println!(" Current: {}", existing_hash); - println!(" New: {}", new_hash); - println!("šŸ”„ Removing outdated snippet..."); - // Remove old snippet file fs::remove_file(snippets_dir.join(&existing_snippet.full_name)).map_err(|e| { Error::new( From 7f29ff9529ba374141b135352b2736bd5e089173 Mon Sep 17 00:00:00 2001 From: Prakash Date: Wed, 20 Nov 2024 02:22:11 +0530 Subject: [PATCH 52/55] made changes to accomodate merge --- macros/src/lib.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index a6fc636..5c03a51 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -10,6 +10,7 @@ use regex::Regex; use std::{ cmp::min, collections::HashMap, + fmt::Display, fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, @@ -522,7 +523,10 @@ fn export_internal( /// Output should match `rustfmt` output exactly. #[proc_macro] pub fn embed(tokens: TokenStream) -> TokenStream { - match embed_internal(tokens, MarkdownLanguage::Ignore) { + match embed_internal( + tokens, + vec![MarkdownLanguage::Rust, MarkdownLanguage::Ignore], + ) { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), } @@ -535,7 +539,7 @@ pub fn embed(tokens: TokenStream) -> TokenStream { /// [`docify::embed!(..)`](`macro@embed`) also apply to this macro. #[proc_macro] pub fn embed_run(tokens: TokenStream) -> TokenStream { - match embed_internal(tokens, MarkdownLanguage::Blank) { + match embed_internal(tokens, vec![MarkdownLanguage::Blank]) { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), } @@ -1250,7 +1254,10 @@ fn source_excerpt<'a, T: ToTokens>( .join("\n")) } -fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) -> Result { +fn embed_internal_str( + tokens: impl Into, + langs: Vec, +) -> Result { let args: EmbedArgs = parse2::(tokens.into())?; // get the root of the crate @@ -1326,7 +1333,7 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - })?; let formatted_content = fix_indentation(&extracted_content); - let output: String = into_example(&formatted_content, lang); + let output: String = into_example(&formatted_content, &langs); Ok(output) } else { @@ -1367,20 +1374,23 @@ fn embed_internal_str(tokens: impl Into, lang: MarkdownLanguage) - for (item, style) in visitor.results { let excerpt = source_excerpt(&source_code, &item, style)?; let formatted = fix_indentation(excerpt); - let example = into_example(formatted.as_str(), lang); + let example = into_example(formatted.as_str(), &langs); results.push(example); } results.join("\n") } else { - into_example(source_code.as_str(), lang) + into_example(source_code.as_str(), &langs) }; Ok(output) } } /// Internal implementation behind [`macro@embed`]. -fn embed_internal(tokens: impl Into, lang: MarkdownLanguage) -> Result { - let output = embed_internal_str(tokens, lang)?; +fn embed_internal( + tokens: impl Into, + langs: Vec, +) -> Result { + let output = embed_internal_str(tokens, langs)?; Ok(quote!(#output)) } From f2178a636c64055828652064e8e56b11e13d14f9 Mon Sep 17 00:00:00 2001 From: Prakash Date: Wed, 20 Nov 2024 02:35:13 +0530 Subject: [PATCH 53/55] fixes --- macros/src/lib.rs | 7 +------ macros/src/tests.rs | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index fb335c8..0acfc56 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -11,7 +11,6 @@ use std::{ cmp::min, collections::HashMap, fmt::Display, - fmt::Display, fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, @@ -524,10 +523,6 @@ fn export_internal( /// Output should match `rustfmt` output exactly. #[proc_macro] pub fn embed(tokens: TokenStream) -> TokenStream { - match embed_internal( - tokens, - vec![MarkdownLanguage::Rust, MarkdownLanguage::Ignore], - ) { match embed_internal( tokens, vec![MarkdownLanguage::Rust, MarkdownLanguage::Ignore], @@ -1329,7 +1324,7 @@ fn embed_internal_str( })?; let formatted_content = fix_indentation(&content); - let output = into_example(&formatted_content, lang); + let output = into_example(&formatted_content, &langs); return Ok(output); } diff --git a/macros/src/tests.rs b/macros/src/tests.rs index b9a298e..10ab79f 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -1,8 +1,4 @@ use super::*; -use proc_macro2::TokenStream as TokenStream2; -use std::fs; -use std::path::PathBuf; -use tempfile::TempDir; #[test] fn test_export_basic_parsing_valid() { From 1bf980c8f045ed276f465b7adbbf2e350100c055 Mon Sep 17 00:00:00 2001 From: Prakash Date: Sat, 14 Dec 2024 09:36:30 +0530 Subject: [PATCH 54/55] all tests passing --- macros/src/lib.rs | 38 ++++++++++++++++++++++++++++++-------- macros/src/tests.rs | 4 ++-- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 5c03a51..57c8eb3 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -905,17 +905,36 @@ enum MarkdownLanguage { Blank, } +impl Display for MarkdownLanguage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MarkdownLanguage::Ignore => write!(f, "ignore"), + MarkdownLanguage::Rust => write!(f, "rust"), + MarkdownLanguage::Blank => write!(f, ""), + } + } +} + /// Converts a source string to a codeblocks wrapped example -fn into_example(st: &str, lang: MarkdownLanguage) -> String { +fn into_example(st: &str, langs: &Vec) -> String { let mut lines: Vec = Vec::new(); - match lang { - MarkdownLanguage::Ignore => lines.push(String::from("```ignore")), - MarkdownLanguage::Rust => lines.push(String::from("```rust")), - MarkdownLanguage::Blank => lines.push(String::from("```")), - } + + // Add the markdown languages (can be more than one, or none) + let mut lang_line = String::from("```"); + lang_line.push_str( + langs + .iter() + .map(|lang| lang.to_string()) + .collect::>() + .join(",") + .as_str(), + ); + lines.push(lang_line); + for line in st.lines() { lines.push(String::from(line)); } + lines.push(String::from("```")); lines.join("\n") } @@ -1305,7 +1324,7 @@ fn embed_internal_str( })?; let formatted_content = fix_indentation(&content); - let output = into_example(&formatted_content, lang); + let output = into_example(&formatted_content, &langs); return Ok(output); } @@ -1606,7 +1625,10 @@ fn compile_markdown_source>(source: S) -> Result { let comment = &orig_comment[4..(orig_comment.len() - 3)].trim(); if comment.starts_with("docify") { let args = parse2::(comment.parse()?)?.args; - let compiled = embed_internal_str(args.to_token_stream(), MarkdownLanguage::Rust)?; + let compiled = embed_internal_str( + args.to_token_stream(), + vec![MarkdownLanguage::Rust, MarkdownLanguage::Ignore], + )?; output.push(compiled); } else { output.push(String::from(orig_comment)); diff --git a/macros/src/tests.rs b/macros/src/tests.rs index d42b428..b9a298e 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -70,7 +70,7 @@ fn test_compile_markdown_valid() { .to_string(), "\"# This is a markdown file\\n\\n```rust\\nstruct \ Something;\\n```\\n\\n\\n`\ - ``rust\\nfn some_fn() {\\n println!(\\\"foo\\\");\ + ``rust,ignore\\nfn some_fn() {\\n println!(\\\"foo\\\");\ \\n}\\n```\\n\\nSome text this is some text\\n\"" ); } @@ -100,7 +100,7 @@ fn test_compile_markdown_source_valid() { "this is some markdown\n\ this is some more markdown\n\ # this is a title\n\ - ```rust\n\ + ```rust,ignore\n\ fn some_fn() {\n \ println!(\"foo\");\n\ }\n\ From 1b696c09c7a424d661d762c559af83071b0ab12b Mon Sep 17 00:00:00 2001 From: Prakash Date: Sat, 14 Dec 2024 09:59:28 +0530 Subject: [PATCH 55/55] fix warnings --- macros/src/tests.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/macros/src/tests.rs b/macros/src/tests.rs index b9a298e..10ab79f 100644 --- a/macros/src/tests.rs +++ b/macros/src/tests.rs @@ -1,8 +1,4 @@ use super::*; -use proc_macro2::TokenStream as TokenStream2; -use std::fs; -use std::path::PathBuf; -use tempfile::TempDir; #[test] fn test_export_basic_parsing_valid() {