From dc52a0dd1e45584a76af9885e2f71ee1261be153 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:52:22 +0000 Subject: [PATCH 01/24] chore: rustfmt --- examples/parts/arrays.rs | 5 +- examples/parts/arrays_of_arrays.rs | 5 +- examples/parts/decode_strict.rs | 5 +- examples/parts/delimiters.rs | 6 +- examples/parts/mixed_arrays.rs | 5 +- examples/parts/objects.rs | 5 +- examples/parts/round_trip.rs | 15 +---- examples/parts/structs.rs | 10 +-- examples/parts/tabular.rs | 5 +- src/cli/main.rs | 29 ++------ src/decode/expansion.rs | 8 +-- src/decode/mod.rs | 5 +- src/decode/parser.rs | 30 ++------- src/decode/scanner.rs | 6 +- src/decode/validation.rs | 5 +- src/encode/folding.rs | 6 +- src/encode/mod.rs | 14 +--- src/encode/writer.rs | 12 +--- src/lib.rs | 99 +++++----------------------- src/tui/app.rs | 35 ++-------- src/tui/components/diff_viewer.rs | 25 ++----- src/tui/components/editor.rs | 10 +-- src/tui/components/file_browser.rs | 26 ++------ src/tui/components/help_screen.rs | 26 ++------ src/tui/components/history_panel.rs | 26 ++------ src/tui/components/repl_panel.rs | 34 ++-------- src/tui/components/settings_panel.rs | 33 ++-------- src/tui/components/stats_bar.rs | 16 +---- src/tui/components/status_bar.rs | 24 ++----- src/tui/events.rs | 6 +- src/tui/keybindings.rs | 6 +- src/tui/mod.rs | 12 +--- src/tui/repl_command.rs | 5 +- src/tui/state/app_state.rs | 15 +---- src/tui/state/file_state.rs | 5 +- src/tui/state/mod.rs | 22 ++----- src/tui/theme.rs | 6 +- src/tui/ui.rs | 35 ++-------- src/types/delimiter.rs | 5 +- src/types/mod.rs | 24 ++----- src/types/options.rs | 6 +- src/types/value.rs | 5 +- src/utils/mod.rs | 18 +---- src/utils/string.rs | 5 +- src/utils/validation.rs | 5 +- tests/arrays.rs | 10 +-- tests/delimiters.rs | 13 +--- tests/errors.rs | 13 +--- tests/numeric.rs | 10 +-- tests/objects.rs | 10 +-- tests/real_world.rs | 10 +-- tests/round_trip.rs | 10 +-- tests/spec_fixtures.rs | 12 +--- tests/unicode.rs | 10 +-- 54 files changed, 140 insertions(+), 668 deletions(-) diff --git a/examples/parts/arrays.rs b/examples/parts/arrays.rs index 7f68638..260a301 100644 --- a/examples/parts/arrays.rs +++ b/examples/parts/arrays.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/arrays_of_arrays.rs b/examples/parts/arrays_of_arrays.rs index 5eb6ae7..bd6c319 100644 --- a/examples/parts/arrays_of_arrays.rs +++ b/examples/parts/arrays_of_arrays.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/decode_strict.rs b/examples/parts/decode_strict.rs index 14303bd..5af6bf9 100644 --- a/examples/parts/decode_strict.rs +++ b/examples/parts/decode_strict.rs @@ -1,8 +1,5 @@ use serde_json::Value; -use toon_format::{ - decode, - DecodeOptions, -}; +use toon_format::{decode, DecodeOptions}; pub fn decode_strict() { // Malformed: header says 2 rows, but only 1 provided diff --git a/examples/parts/delimiters.rs b/examples/parts/delimiters.rs index b44eeaa..338e7ee 100644 --- a/examples/parts/delimiters.rs +++ b/examples/parts/delimiters.rs @@ -1,9 +1,5 @@ use serde_json::json; -use toon_format::{ - encode, - Delimiter, - EncodeOptions, -}; +use toon_format::{encode, Delimiter, EncodeOptions}; pub fn delimiters() { let data = json!({ diff --git a/examples/parts/mixed_arrays.rs b/examples/parts/mixed_arrays.rs index c3e120b..3cf4afd 100644 --- a/examples/parts/mixed_arrays.rs +++ b/examples/parts/mixed_arrays.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/objects.rs b/examples/parts/objects.rs index d7b1bca..da6769f 100644 --- a/examples/parts/objects.rs +++ b/examples/parts/objects.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/round_trip.rs b/examples/parts/round_trip.rs index 5c5c3bd..b5873a8 100644 --- a/examples/parts/round_trip.rs +++ b/examples/parts/round_trip.rs @@ -1,15 +1,6 @@ -use serde::{ - Deserialize, - Serialize, -}; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct Product { diff --git a/examples/parts/structs.rs b/examples/parts/structs.rs index 7b49e48..14ec066 100644 --- a/examples/parts/structs.rs +++ b/examples/parts/structs.rs @@ -1,11 +1,5 @@ -use serde::{ - Deserialize, - Serialize, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde::{Deserialize, Serialize}; +use toon_format::{decode_default, encode_default}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct User { diff --git a/examples/parts/tabular.rs b/examples/parts/tabular.rs index 1e210ca..c93ba08 100644 --- a/examples/parts/tabular.rs +++ b/examples/parts/tabular.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/src/cli/main.rs b/src/cli/main.rs index ecc8e9f..311fb00 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -1,36 +1,17 @@ use std::{ fs, - io::{ - self, - Read, - Write, - }, - path::{ - Path, - PathBuf, - }, + io::{self, Read, Write}, + path::{Path, PathBuf}, }; -use anyhow::{ - bail, - Context, - Result, -}; +use anyhow::{bail, Context, Result}; use clap::Parser; use comfy_table::Table; use serde::Serialize; use tiktoken_rs::cl100k_base; use toon_format::{ - decode, - encode, - types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + decode, encode, + types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}, }; #[derive(Parser, Debug)] diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index cd645bc..ff31875 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -2,13 +2,7 @@ use indexmap::IndexMap; use crate::{ constants::QUOTED_KEY_MARKER, - types::{ - is_identifier_segment, - JsonValue as Value, - PathExpansionMode, - ToonError, - ToonResult, - }, + types::{is_identifier_segment, JsonValue as Value, PathExpansionMode, ToonError, ToonResult}, }; pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option> { diff --git a/src/decode/mod.rs b/src/decode/mod.rs index 574b3a7..420055c 100644 --- a/src/decode/mod.rs +++ b/src/decode/mod.rs @@ -6,10 +6,7 @@ pub mod validation; use serde_json::Value; -use crate::types::{ - DecodeOptions, - ToonResult, -}; +use crate::types::{DecodeOptions, ToonResult}; /// Decode a TOON string into any deserializable type. /// diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 0f40084..77f8fa6 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -1,29 +1,12 @@ -use serde_json::{ - Map, - Number, - Value, -}; +use serde_json::{Map, Number, Value}; use crate::{ - constants::{ - KEYWORDS, - MAX_DEPTH, - QUOTED_KEY_MARKER, - }, + constants::{KEYWORDS, MAX_DEPTH, QUOTED_KEY_MARKER}, decode::{ - scanner::{ - Scanner, - Token, - }, + scanner::{Scanner, Token}, validation, }, - types::{ - DecodeOptions, - Delimiter, - ErrorContext, - ToonError, - ToonResult, - }, + types::{DecodeOptions, Delimiter, ErrorContext, ToonError, ToonResult}, utils::validation::validate_depth, }; @@ -1441,10 +1424,7 @@ mod tests { #[test] fn test_round_trip_parentheses() { - use crate::{ - decode::decode_default, - encode::encode_default, - }; + use crate::{decode::decode_default, encode::encode_default}; let original = json!({ "message": "Mostly Functions (3 of 3)", diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 2301330..a04247e 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -1,8 +1,4 @@ -use crate::types::{ - Delimiter, - ToonError, - ToonResult, -}; +use crate::types::{Delimiter, ToonError, ToonResult}; /// Tokens produced by the scanner during lexical analysis. #[derive(Debug, Clone, PartialEq)] diff --git a/src/decode/validation.rs b/src/decode/validation.rs index 40608ef..9ff36b9 100644 --- a/src/decode/validation.rs +++ b/src/decode/validation.rs @@ -1,7 +1,4 @@ -use crate::types::{ - ToonError, - ToonResult, -}; +use crate::types::{ToonError, ToonResult}; /// Validate that array length matches expected value. pub fn validate_array_length(expected: usize, actual: usize) -> ToonResult<()> { diff --git a/src/encode/folding.rs b/src/encode/folding.rs index beec7ae..0034970 100644 --- a/src/encode/folding.rs +++ b/src/encode/folding.rs @@ -1,8 +1,4 @@ -use crate::types::{ - is_identifier_segment, - JsonValue as Value, - KeyFoldingMode, -}; +use crate::types::{is_identifier_segment, JsonValue as Value, KeyFoldingMode}; /// Result of chain analysis for folding. pub struct FoldableChain { diff --git a/src/encode/mod.rs b/src/encode/mod.rs index 0a11987..56dd24c 100644 --- a/src/encode/mod.rs +++ b/src/encode/mod.rs @@ -7,19 +7,9 @@ use indexmap::IndexMap; use crate::{ constants::MAX_DEPTH, types::{ - EncodeOptions, - IntoJsonValue, - JsonValue as Value, - KeyFoldingMode, - ToonError, - ToonResult, - }, - utils::{ - format_canonical_number, - normalize, - validation::validate_depth, - QuotingContext, + EncodeOptions, IntoJsonValue, JsonValue as Value, KeyFoldingMode, ToonError, ToonResult, }, + utils::{format_canonical_number, normalize, validation::validate_depth, QuotingContext}, }; /// Encode any serializable value to TOON format. diff --git a/src/encode/writer.rs b/src/encode/writer.rs index ba8ad96..7edf35e 100644 --- a/src/encode/writer.rs +++ b/src/encode/writer.rs @@ -1,15 +1,7 @@ use crate::{ - types::{ - Delimiter, - EncodeOptions, - ToonResult, - }, + types::{Delimiter, EncodeOptions, ToonResult}, utils::{ - string::{ - is_valid_unquoted_key, - needs_quoting, - quote_string, - }, + string::{is_valid_unquoted_key, needs_quoting, quote_string}, QuotingContext, }, }; diff --git a/src/lib.rs b/src/lib.rs index 67c0133..ce59d22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,66 +34,27 @@ pub mod types; pub mod utils; pub use decode::{ - decode, - decode_default, - decode_no_coerce, - decode_no_coerce_with_options, - decode_strict, + decode, decode_default, decode_no_coerce, decode_no_coerce_with_options, decode_strict, decode_strict_with_options, }; -pub use encode::{ - encode, - encode_array, - encode_default, - encode_object, -}; -pub use types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - ToonError, -}; +pub use encode::{encode, encode_array, encode_default, encode_object}; +pub use types::{DecodeOptions, Delimiter, EncodeOptions, Indent, ToonError}; pub use utils::{ - literal::{ - is_keyword, - is_literal_like, - }, + literal::{is_keyword, is_literal_like}, normalize, - string::{ - escape_string, - is_valid_unquoted_key, - needs_quoting, - }, + string::{escape_string, is_valid_unquoted_key, needs_quoting}, }; #[cfg(test)] mod tests { - use serde_json::{ - json, - Value, - }; + use serde_json::{json, Value}; use crate::{ constants::is_keyword, - decode::{ - decode_default, - decode_strict, - }, - encode::{ - encode, - encode_default, - }, - types::{ - Delimiter, - EncodeOptions, - }, - utils::{ - escape_string, - is_literal_like, - needs_quoting, - normalize, - }, + decode::{decode_default, decode_strict}, + encode::{encode, encode_default}, + types::{Delimiter, EncodeOptions}, + utils::{escape_string, is_literal_like, needs_quoting, normalize}, }; #[test] @@ -160,10 +121,7 @@ mod tests { assert!(needs_quoting("true", Delimiter::Comma.as_char())); } - use serde::{ - Deserialize, - Serialize, - }; + use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct TestUser { @@ -174,10 +132,7 @@ mod tests { #[test] fn test_encode_decode_simple_struct() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let user = TestUser { name: "Alice".to_string(), @@ -203,10 +158,7 @@ mod tests { #[test] fn test_encode_decode_with_array() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let product = TestProduct { id: 42, @@ -221,10 +173,7 @@ mod tests { #[test] fn test_encode_decode_vec_of_structs() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let users = vec![ TestUser { @@ -262,10 +211,7 @@ mod tests { #[test] fn test_encode_decode_nested_structs() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let nested = Nested { outer: OuterStruct { @@ -283,10 +229,7 @@ mod tests { #[test] fn test_round_trip_list_item_tabular_v3() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let original = json!({ "items": [ @@ -309,10 +252,7 @@ mod tests { #[test] fn test_round_trip_complex_list_item_tabular_v3() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let original = json!({ "data": [ @@ -342,10 +282,7 @@ mod tests { #[test] fn test_round_trip_mixed_list_items_v3() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let original = json!({ "entries": [ diff --git a/src/tui/app.rs b/src/tui/app.rs index b5bbb6b..7b23c31 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -1,39 +1,18 @@ -use std::{ - fs, - path::PathBuf, - time::Duration, -}; +use std::{fs, path::PathBuf, time::Duration}; -use anyhow::{ - Context, - Result, -}; +use anyhow::{Context, Result}; use chrono::Local; -use crossterm::event::{ - KeyCode, - KeyEvent, -}; +use crossterm::event::{KeyCode, KeyEvent}; use tiktoken_rs::cl100k_base; use crate::{ - decode, - encode, + decode, encode, tui::{ components::FileBrowser, - events::{ - Event, - EventHandler, - }, - keybindings::{ - Action, - KeyBindings, - }, + events::{Event, EventHandler}, + keybindings::{Action, KeyBindings}, repl_command::ReplCommand, - state::{ - app_state::ConversionStats, - AppState, - ConversionHistory, - }, + state::{app_state::ConversionStats, AppState, ConversionHistory}, ui, }, }; diff --git a/src/tui/components/diff_viewer.rs b/src/tui/components/diff_viewer.rs index 516e73d..9494fb8 100644 --- a/src/tui/components/diff_viewer.rs +++ b/src/tui/components/diff_viewer.rs @@ -1,30 +1,13 @@ //! Side-by-side diff viewer for input/output comparison. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - Wrap, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Wrap}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct DiffViewer; diff --git a/src/tui/components/editor.rs b/src/tui/components/editor.rs index 6171f21..00fb043 100644 --- a/src/tui/components/editor.rs +++ b/src/tui/components/editor.rs @@ -2,17 +2,11 @@ use ratatui::{ layout::Rect, - widgets::{ - Block, - Borders, - }, + widgets::{Block, Borders}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct EditorComponent; diff --git a/src/tui/components/file_browser.rs b/src/tui/components/file_browser.rs index 52baf55..4baf080 100644 --- a/src/tui/components/file_browser.rs +++ b/src/tui/components/file_browser.rs @@ -3,31 +3,13 @@ use std::fs; use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; /// File browser state and rendering. pub struct FileBrowser { diff --git a/src/tui/components/help_screen.rs b/src/tui/components/help_screen.rs index 2f2be40..8cfe995 100644 --- a/src/tui/components/help_screen.rs +++ b/src/tui/components/help_screen.rs @@ -1,31 +1,13 @@ //! Help screen showing keyboard shortcuts. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; -use crate::tui::{ - keybindings::KeyBindings, - theme::Theme, -}; +use crate::tui::{keybindings::KeyBindings, theme::Theme}; pub struct HelpScreen; diff --git a/src/tui/components/history_panel.rs b/src/tui/components/history_panel.rs index 9bdc945..764a09d 100644 --- a/src/tui/components/history_panel.rs +++ b/src/tui/components/history_panel.rs @@ -1,31 +1,13 @@ //! Conversion history panel. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct HistoryPanel; diff --git a/src/tui/components/repl_panel.rs b/src/tui/components/repl_panel.rs index 681a566..5acb36c 100644 --- a/src/tui/components/repl_panel.rs +++ b/src/tui/components/repl_panel.rs @@ -1,36 +1,12 @@ use ratatui::{ - layout::{ - Constraint, - Direction, - Layout, - Margin, - Rect, - }, - style::{ - Color, - Modifier, - Style, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - Scrollbar, - ScrollbarOrientation, - ScrollbarState, - Wrap, - }, + layout::{Constraint, Direction, Layout, Margin, Rect}, + style::{Color, Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Scrollbar, ScrollbarOrientation, ScrollbarState, Wrap}, Frame, }; -use crate::tui::state::{ - AppState, - ReplLineKind, -}; +use crate::tui::state::{AppState, ReplLineKind}; pub struct ReplPanel; diff --git a/src/tui/components/settings_panel.rs b/src/tui/components/settings_panel.rs index c8d84c1..a7debad 100644 --- a/src/tui/components/settings_panel.rs +++ b/src/tui/components/settings_panel.rs @@ -1,38 +1,15 @@ //! Settings panel for configuring encode/decode options. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; use crate::{ - tui::{ - state::AppState, - theme::Theme, - }, - types::{ - Delimiter, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + tui::{state::AppState, theme::Theme}, + types::{Delimiter, Indent, KeyFoldingMode, PathExpansionMode}, }; pub struct SettingsPanel; diff --git a/src/tui/components/stats_bar.rs b/src/tui/components/stats_bar.rs index 3691d9b..4afb0e8 100644 --- a/src/tui/components/stats_bar.rs +++ b/src/tui/components/stats_bar.rs @@ -2,22 +2,12 @@ use ratatui::{ layout::Rect, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - }, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct StatsBar; diff --git a/src/tui/components/status_bar.rs b/src/tui/components/status_bar.rs index 8d1fb00..999f650 100644 --- a/src/tui/components/status_bar.rs +++ b/src/tui/components/status_bar.rs @@ -1,29 +1,13 @@ //! Status bar showing mode, file, and key commands. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct StatusBar; diff --git a/src/tui/events.rs b/src/tui/events.rs index 64a6e22..0d22c91 100644 --- a/src/tui/events.rs +++ b/src/tui/events.rs @@ -2,11 +2,7 @@ use std::time::Duration; -use crossterm::event::{ - self, - Event as CrosstermEvent, - KeyEvent, -}; +use crossterm::event::{self, Event as CrosstermEvent, KeyEvent}; /// TUI events. pub enum Event { diff --git a/src/tui/keybindings.rs b/src/tui/keybindings.rs index e82ccb1..f755e02 100644 --- a/src/tui/keybindings.rs +++ b/src/tui/keybindings.rs @@ -1,10 +1,6 @@ //! Keyboard shortcuts and action mapping. -use crossterm::event::{ - KeyCode, - KeyEvent, - KeyModifiers, -}; +use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; /// Actions that can be triggered by keyboard shortcuts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/tui/mod.rs b/src/tui/mod.rs index faa352b..a7037ed 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -18,17 +18,9 @@ use anyhow::Result; pub use app::TuiApp; use crossterm::{ execute, - terminal::{ - disable_raw_mode, - enable_raw_mode, - EnterAlternateScreen, - LeaveAlternateScreen, - }, -}; -use ratatui::{ - backend::CrosstermBackend, - Terminal, + terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}, }; +use ratatui::{backend::CrosstermBackend, Terminal}; /// Initialize and run the TUI application. /// diff --git a/src/tui/repl_command.rs b/src/tui/repl_command.rs index 5191f2f..33e15ee 100644 --- a/src/tui/repl_command.rs +++ b/src/tui/repl_command.rs @@ -1,9 +1,6 @@ //! REPL command parser with inline data support -use anyhow::{ - bail, - Result, -}; +use anyhow::{bail, Result}; /// Parsed REPL command with inline data #[derive(Debug, Clone)] diff --git a/src/tui/state/app_state.rs b/src/tui/state/app_state.rs index 7c70b32..d4925b8 100644 --- a/src/tui/state/app_state.rs +++ b/src/tui/state/app_state.rs @@ -1,20 +1,9 @@ //! Main application state. -use super::{ - EditorState, - FileState, - ReplState, -}; +use super::{EditorState, FileState, ReplState}; use crate::{ tui::theme::Theme, - types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}, }; /// Conversion mode (encode/decode). diff --git a/src/tui/state/file_state.rs b/src/tui/state/file_state.rs index 436e971..6467381 100644 --- a/src/tui/state/file_state.rs +++ b/src/tui/state/file_state.rs @@ -2,10 +2,7 @@ use std::path::PathBuf; -use chrono::{ - DateTime, - Local, -}; +use chrono::{DateTime, Local}; /// A file or directory entry. #[derive(Debug, Clone)] diff --git a/src/tui/state/mod.rs b/src/tui/state/mod.rs index 9943270..def4e5f 100644 --- a/src/tui/state/mod.rs +++ b/src/tui/state/mod.rs @@ -5,21 +5,7 @@ pub mod editor_state; pub mod file_state; pub mod repl_state; -pub use app_state::{ - AppState, - ConversionStats, - Mode, -}; -pub use editor_state::{ - EditorMode, - EditorState, -}; -pub use file_state::{ - ConversionHistory, - FileState, -}; -pub use repl_state::{ - ReplLine, - ReplLineKind, - ReplState, -}; +pub use app_state::{AppState, ConversionStats, Mode}; +pub use editor_state::{EditorMode, EditorState}; +pub use file_state::{ConversionHistory, FileState}; +pub use repl_state::{ReplLine, ReplLineKind, ReplState}; diff --git a/src/tui/theme.rs b/src/tui/theme.rs index 8d436d0..3cf5e56 100644 --- a/src/tui/theme.rs +++ b/src/tui/theme.rs @@ -1,10 +1,6 @@ //! Color themes for the TUI. -use ratatui::style::{ - Color, - Modifier, - Style, -}; +use ratatui::style::{Color, Modifier, Style}; /// Available color themes. #[derive(Debug, Clone, Copy, PartialEq, Default)] diff --git a/src/tui/ui.rs b/src/tui/ui.rs index 2910c37..22e3ae2 100644 --- a/src/tui/ui.rs +++ b/src/tui/ui.rs @@ -1,42 +1,19 @@ use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, Frame, }; use super::{ components::{ - DiffViewer, - EditorComponent, - FileBrowser, - HelpScreen, - HistoryPanel, - ReplPanel, - SettingsPanel, - StatsBar, - StatusBar, + DiffViewer, EditorComponent, FileBrowser, HelpScreen, HistoryPanel, ReplPanel, + SettingsPanel, StatsBar, StatusBar, }, state::AppState, theme::Theme, }; -use crate::types::{ - KeyFoldingMode, - PathExpansionMode, -}; +use crate::types::{KeyFoldingMode, PathExpansionMode}; /// Main render function - orchestrates all UI components. pub fn render(f: &mut Frame, app: &mut AppState, file_browser: &mut FileBrowser) { diff --git a/src/types/delimiter.rs b/src/types/delimiter.rs index e554e6a..cb387a5 100644 --- a/src/types/delimiter.rs +++ b/src/types/delimiter.rs @@ -1,9 +1,6 @@ use std::fmt; -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; /// Delimiter character used to separate array elements. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] diff --git a/src/types/mod.rs b/src/types/mod.rs index 42b524f..ebfaf68 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -5,23 +5,7 @@ mod options; mod value; pub use delimiter::Delimiter; -pub use errors::{ - ErrorContext, - ToonError, - ToonResult, -}; -pub use folding::{ - is_identifier_segment, - KeyFoldingMode, - PathExpansionMode, -}; -pub use options::{ - DecodeOptions, - EncodeOptions, - Indent, -}; -pub use value::{ - IntoJsonValue, - JsonValue, - Number, -}; +pub use errors::{ErrorContext, ToonError, ToonResult}; +pub use folding::{is_identifier_segment, KeyFoldingMode, PathExpansionMode}; +pub use options::{DecodeOptions, EncodeOptions, Indent}; +pub use value::{IntoJsonValue, JsonValue, Number}; diff --git a/src/types/options.rs b/src/types/options.rs index 5e1ecb3..ec4562c 100644 --- a/src/types/options.rs +++ b/src/types/options.rs @@ -1,10 +1,6 @@ use crate::{ constants::DEFAULT_INDENT, - types::{ - Delimiter, - KeyFoldingMode, - PathExpansionMode, - }, + types::{Delimiter, KeyFoldingMode, PathExpansionMode}, }; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/types/value.rs b/src/types/value.rs index 15885cb..568302d 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -1,9 +1,6 @@ use std::{ fmt, - ops::{ - Index, - IndexMut, - }, + ops::{Index, IndexMut}, }; use indexmap::IndexMap; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index cf355c6..1b300cb 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -4,25 +4,13 @@ pub mod string; pub mod validation; use indexmap::IndexMap; -pub use literal::{ - is_keyword, - is_literal_like, - is_numeric_like, - is_structural_char, -}; +pub use literal::{is_keyword, is_literal_like, is_numeric_like, is_structural_char}; pub use number::format_canonical_number; pub use string::{ - escape_string, - is_valid_unquoted_key, - needs_quoting, - quote_string, - unescape_string, + escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, }; -use crate::types::{ - JsonValue as Value, - Number, -}; +use crate::types::{JsonValue as Value, Number}; /// Context for determining when quoting is needed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/utils/string.rs b/src/utils/string.rs index 7423d5a..3622990 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -1,7 +1,4 @@ -use crate::{ - types::Delimiter, - utils::literal, -}; +use crate::{types::Delimiter, utils::literal}; /// Escape special characters in a string for quoted output. pub fn escape_string(s: &str) -> String { diff --git a/src/utils/validation.rs b/src/utils/validation.rs index d14167b..72cdf84 100644 --- a/src/utils/validation.rs +++ b/src/utils/validation.rs @@ -1,9 +1,6 @@ use serde_json::Value; -use crate::types::{ - ToonError, - ToonResult, -}; +use crate::types::{ToonError, ToonResult}; /// Validate that nesting depth doesn't exceed the maximum. pub fn validate_depth(depth: usize, max_depth: usize) -> ToonResult<()> { diff --git a/tests/arrays.rs b/tests/arrays.rs index 437fafa..b879f52 100644 --- a/tests/arrays.rs +++ b/tests/arrays.rs @@ -1,13 +1,7 @@ use std::f64; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_tabular_arrays() { diff --git a/tests/delimiters.rs b/tests/delimiters.rs index 7d1bb28..7e52a67 100644 --- a/tests/delimiters.rs +++ b/tests/delimiters.rs @@ -1,14 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode, - encode_default, - Delimiter, - EncodeOptions, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode, encode_default, Delimiter, EncodeOptions}; #[test] fn test_delimiter_variants() { diff --git a/tests/errors.rs b/tests/errors.rs index 935b567..a100332 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -1,14 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode, - decode_default, - decode_strict, - DecodeOptions, - ToonError, -}; +use serde_json::{json, Value}; +use toon_format::{decode, decode_default, decode_strict, DecodeOptions, ToonError}; #[test] fn test_invalid_syntax_errors() { diff --git a/tests/numeric.rs b/tests/numeric.rs index 18f4679..432c43c 100644 --- a/tests/numeric.rs +++ b/tests/numeric.rs @@ -1,13 +1,7 @@ use core::f64; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_numeric_edge_cases() { diff --git a/tests/objects.rs b/tests/objects.rs index f73446b..7709b74 100644 --- a/tests/objects.rs +++ b/tests/objects.rs @@ -1,11 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_special_characters_and_quoting() { diff --git a/tests/real_world.rs b/tests/real_world.rs index d47471f..708df22 100644 --- a/tests/real_world.rs +++ b/tests/real_world.rs @@ -1,11 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_real_world_github_data() { diff --git a/tests/round_trip.rs b/tests/round_trip.rs index 3125276..c40dea0 100644 --- a/tests/round_trip.rs +++ b/tests/round_trip.rs @@ -1,13 +1,7 @@ use std::f64; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_comprehensive_round_trips() { diff --git a/tests/spec_fixtures.rs b/tests/spec_fixtures.rs index 17c18f7..7d4a308 100644 --- a/tests/spec_fixtures.rs +++ b/tests/spec_fixtures.rs @@ -2,16 +2,8 @@ use datatest_stable::Utf8Path; use serde::Deserialize; use serde_json::Value; use toon_format::{ - decode, - encode, - types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + decode, encode, + types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}, }; #[derive(Deserialize, Debug)] diff --git a/tests/unicode.rs b/tests/unicode.rs index 9c7f35d..447e709 100644 --- a/tests/unicode.rs +++ b/tests/unicode.rs @@ -1,11 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_unicode_strings() { From 8dbf842106f7442fc3be4283c96fb1390f001eae Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:54:46 +0000 Subject: [PATCH 02/24] fix(types): add safe get methods to JsonValue --- src/types/value.rs | 60 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/src/types/value.rs b/src/types/value.rs index 568302d..d81532b 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -324,6 +324,20 @@ impl JsonValue { } } + pub fn get(&self, key: &str) -> Option<&JsonValue> { + match self { + JsonValue::Object(obj) => obj.get(key), + _ => None, + } + } + + pub fn get_index(&self, index: usize) -> Option<&JsonValue> { + match self { + JsonValue::Array(arr) => arr.get(index), + _ => None, + } + } + /// Takes the value, leaving Null in its place. pub fn take(&mut self) -> JsonValue { std::mem::replace(self, JsonValue::Null) @@ -377,8 +391,16 @@ impl Index for JsonValue { fn index(&self, index: usize) -> &Self::Output { match self { - JsonValue::Array(arr) => &arr[index], - _ => panic!("cannot index into non-array value with usize"), + JsonValue::Array(arr) => arr.get(index).unwrap_or_else(|| { + panic!( + "index {index} out of bounds for array of length {}", + arr.len() + ) + }), + _ => panic!( + "cannot index into non-array value of type {}", + self.type_name() + ), } } } @@ -386,8 +408,16 @@ impl Index for JsonValue { impl IndexMut for JsonValue { fn index_mut(&mut self, index: usize) -> &mut Self::Output { match self { - JsonValue::Array(arr) => &mut arr[index], - _ => panic!("cannot index into non-array value with usize"), + JsonValue::Array(arr) => arr.get_mut(index).unwrap_or_else(|| { + panic!( + "index {index} out of bounds for array of length {}", + arr.len() + ) + }), + _ => panic!( + "cannot index into non-array value of type {}", + self.type_name() + ), } } } @@ -397,10 +427,13 @@ impl Index<&str> for JsonValue { fn index(&self, key: &str) -> &Self::Output { match self { - JsonValue::Object(obj) => obj - .get(key) - .unwrap_or_else(|| panic!("key '{key}' not found in object")), - _ => panic!("cannot index into non-object value with &str"), + JsonValue::Object(obj) => obj.get(key).unwrap_or_else(|| { + panic!("key '{key}' not found in object with {} entries", obj.len()) + }), + _ => panic!( + "cannot index into non-object value of type {}", + self.type_name() + ), } } } @@ -408,10 +441,13 @@ impl Index<&str> for JsonValue { impl IndexMut<&str> for JsonValue { fn index_mut(&mut self, key: &str) -> &mut Self::Output { match self { - JsonValue::Object(obj) => obj - .get_mut(key) - .unwrap_or_else(|| panic!("key '{key}' not found in object")), - _ => panic!("cannot index into non-object value with &str"), + JsonValue::Object(obj) => obj.get_mut(key).unwrap_or_else(|| { + panic!("key '{key}' not found in object with {} entries", obj.len()) + }), + _ => panic!( + "cannot index into non-object value of type {}", + self.type_name() + ), } } } From e69c69d2a630a194c8332681ceeaa238f063dcc6 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:00 +0000 Subject: [PATCH 03/24] fix(scanner): replace unwrap with safe option handling --- src/decode/scanner.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index a04247e..2631b88 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -348,9 +348,10 @@ impl Scanner { // Leading zeros like "05" are strings, but "0", "0.5", "-0" are numbers if s.starts_with('0') && s.len() > 1 { - let second_char = s.chars().nth(1).unwrap(); - if second_char.is_ascii_digit() { - return Ok(Token::String(s.to_string(), false)); + if let Some(second_char) = s.chars().nth(1) { + if second_char.is_ascii_digit() { + return Ok(Token::String(s.to_string(), false)); + } } } @@ -454,12 +455,14 @@ impl Scanner { _ => {} } - if trimmed.starts_with('-') || trimmed.chars().next().unwrap().is_ascii_digit() { + if trimmed.starts_with('-') || trimmed.chars().next().map_or(false, |c| c.is_ascii_digit()) + { // Leading zeros like "05" are strings if trimmed.starts_with('0') && trimmed.len() > 1 { - let second_char = trimmed.chars().nth(1).unwrap(); - if second_char.is_ascii_digit() { - return Ok(Token::String(trimmed.to_string(), false)); + if let Some(second_char) = trimmed.chars().nth(1) { + if second_char.is_ascii_digit() { + return Ok(Token::String(trimmed.to_string(), false)); + } } } From 29071099a7f4e3f78caeb99bc01eaa0abc04dcb0 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:17 +0000 Subject: [PATCH 04/24] fix(expansion): use expect with descriptive message --- src/decode/expansion.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index ff31875..3a9d080 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -84,7 +84,7 @@ pub fn deep_merge_value( } } else { target.insert(first_key.clone(), Value::Object(IndexMap::new())); - match target.get_mut(first_key).unwrap() { + match target.get_mut(first_key).expect("key was just inserted") { Value::Object(obj) => obj, _ => unreachable!(), } From 32dfc85d435ed107f277330846d490609645db24 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:37 +0000 Subject: [PATCH 05/24] fix(encode): use expect for field list lookup --- src/encode/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encode/mod.rs b/src/encode/mod.rs index 56dd24c..69b2d1b 100644 --- a/src/encode/mod.rs +++ b/src/encode/mod.rs @@ -215,7 +215,7 @@ fn write_object_impl( writer.write_newline()?; } - let value = &obj[*key]; + let value = obj.get(*key).expect("key exists in field list"); // Check if this key-value pair can be folded (v1.5 feature) // Don't fold if any sibling key is a dotted path starting with this key From 518e15c48d2a21f0c3341a4a38635b0a46debdc9 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:55 +0000 Subject: [PATCH 06/24] test: add panic safety tests --- tests/panic_safety.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/panic_safety.rs diff --git a/tests/panic_safety.rs b/tests/panic_safety.rs new file mode 100644 index 0000000..4fe26d8 --- /dev/null +++ b/tests/panic_safety.rs @@ -0,0 +1,27 @@ +//! Tests to verify panic safety of JsonValue operations + +use toon_format::JsonValue; + +#[test] +fn test_get_missing_key_returns_none() { + let obj = JsonValue::Object(Default::default()); + assert!(obj.get("nonexistent").is_none()); +} + +#[test] +fn test_get_on_non_object_returns_none() { + let arr = JsonValue::Array(vec![]); + assert!(arr.get("key").is_none()); +} + +#[test] +fn test_get_index_out_of_bounds_returns_none() { + let arr = JsonValue::Array(vec![]); + assert!(arr.get_index(0).is_none()); +} + +#[test] +fn test_get_index_on_non_array_returns_none() { + let obj = JsonValue::Object(Default::default()); + assert!(obj.get_index(0).is_none()); +} From 08f5d359f438abe6bea6ec4ea0553911f550d8ef Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:59:00 +0000 Subject: [PATCH 07/24] fix(types): avoid borrow overlap in IndexMut --- src/types/value.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/types/value.rs b/src/types/value.rs index d81532b..5a86e21 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -408,12 +408,12 @@ impl Index for JsonValue { impl IndexMut for JsonValue { fn index_mut(&mut self, index: usize) -> &mut Self::Output { match self { - JsonValue::Array(arr) => arr.get_mut(index).unwrap_or_else(|| { - panic!( - "index {index} out of bounds for array of length {}", - arr.len() - ) - }), + JsonValue::Array(arr) => { + let len = arr.len(); + arr.get_mut(index).unwrap_or_else(|| { + panic!("index {index} out of bounds for array of length {len}") + }) + } _ => panic!( "cannot index into non-array value of type {}", self.type_name() @@ -441,9 +441,12 @@ impl Index<&str> for JsonValue { impl IndexMut<&str> for JsonValue { fn index_mut(&mut self, key: &str) -> &mut Self::Output { match self { - JsonValue::Object(obj) => obj.get_mut(key).unwrap_or_else(|| { - panic!("key '{key}' not found in object with {} entries", obj.len()) - }), + JsonValue::Object(obj) => { + let len = obj.len(); + obj.get_mut(key).unwrap_or_else(|| { + panic!("key '{key}' not found in object with {len} entries") + }) + } _ => panic!( "cannot index into non-object value of type {}", self.type_name() From 31f75b10405d13edcd521cc2667515482c21f1a7 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:59:10 +0000 Subject: [PATCH 08/24] test: fix panic safety import --- tests/panic_safety.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/panic_safety.rs b/tests/panic_safety.rs index 4fe26d8..8630670 100644 --- a/tests/panic_safety.rs +++ b/tests/panic_safety.rs @@ -1,6 +1,6 @@ //! Tests to verify panic safety of JsonValue operations -use toon_format::JsonValue; +use toon_format::types::JsonValue; #[test] fn test_get_missing_key_returns_none() { From 071a27cb935712fd43e0e59534f6c7e6996cfee9 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 13:01:54 +0000 Subject: [PATCH 09/24] fix(scanner): satisfy clippy --- src/decode/scanner.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 2631b88..9ca7615 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -455,7 +455,11 @@ impl Scanner { _ => {} } - if trimmed.starts_with('-') || trimmed.chars().next().map_or(false, |c| c.is_ascii_digit()) + if trimmed.starts_with('-') + || trimmed + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) { // Leading zeros like "05" are strings if trimmed.starts_with('0') && trimmed.len() > 1 { From 67aca3ccebbc5e25f5e04901138bfc2566d6d1ed Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 14:02:12 +0000 Subject: [PATCH 10/24] fix(decoder): improve decode correctness --- src/decode/expansion.rs | 6 +- src/decode/parser.rs | 387 ++++++++++++++++++++++++++++++---------- src/decode/scanner.rs | 72 +++++--- src/types/folding.rs | 4 +- src/utils/mod.rs | 1 + src/utils/string.rs | 19 +- 6 files changed, 361 insertions(+), 128 deletions(-) diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index 3a9d080..d665d45 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -102,10 +102,8 @@ pub fn expand_paths_in_object( let mut result = IndexMap::new(); for (key, mut value) in obj { - // Expand nested objects first (depth-first) - if let Value::Object(nested_obj) = value { - value = Value::Object(expand_paths_in_object(nested_obj, mode, strict)?); - } + // Expand nested structures (arrays/objects) first (depth-first) + value = expand_paths_recursive(value, mode, strict)?; // Strip marker from quoted keys let clean_key = if key.starts_with(QUOTED_KEY_MARKER) { diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 77f8fa6..a682a9d 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -7,7 +7,7 @@ use crate::{ validation, }, types::{DecodeOptions, Delimiter, ErrorContext, ToonError, ToonResult}, - utils::validation::validate_depth, + utils::{is_valid_unquoted_key_decode, validation::validate_depth}, }; /// Context for parsing arrays to determine correct indentation depth. @@ -32,6 +32,7 @@ pub struct Parser<'a> { current_token: Token, options: DecodeOptions, delimiter: Option, + delimiter_stack: Vec>, input: &'a str, } @@ -41,12 +42,14 @@ impl<'a> Parser<'a> { let mut scanner = Scanner::new(input); let chosen_delim = options.delimiter; scanner.set_active_delimiter(chosen_delim); + scanner.set_coerce_types(options.coerce_types); let current_token = scanner.scan_token()?; Ok(Self { scanner, current_token, delimiter: chosen_delim, + delimiter_stack: Vec::new(), options, input, }) @@ -86,6 +89,87 @@ impl<'a> Parser<'a> { Ok(()) } + fn push_delimiter(&mut self, delimiter: Option) { + self.delimiter_stack.push(self.delimiter); + self.delimiter = delimiter; + self.scanner.set_active_delimiter(delimiter); + } + + fn pop_delimiter(&mut self) { + if let Some(previous) = self.delimiter_stack.pop() { + self.delimiter = previous; + self.scanner.set_active_delimiter(previous); + if let (Some(delim), Token::String(value, was_quoted)) = (previous, &self.current_token) + { + if !*was_quoted && value.len() == 1 && value.starts_with(delim.as_char()) { + self.current_token = Token::Delimiter(delim); + } + } + } + } + + fn format_key(&self, key: &str, was_quoted: bool) -> String { + if was_quoted && key.contains('.') { + format!("{QUOTED_KEY_MARKER}{key}") + } else { + key.to_string() + } + } + + fn validate_unquoted_key(&self, key: &str, was_quoted: bool) -> ToonResult<()> { + if self.options.strict && !was_quoted && !is_valid_unquoted_key_decode(key) { + return Err(self + .parse_error_with_context(format!("Invalid unquoted key: '{key}'")) + .with_suggestion("Quote the key to use special characters")); + } + Ok(()) + } + + fn validate_unquoted_string(&self, value: &str, was_quoted: bool) -> ToonResult<()> { + if self.options.strict && !was_quoted && value.contains('\t') { + return Err(self + .parse_error_with_context("Unquoted tab characters are not allowed in strict mode") + .with_suggestion("Quote the value to include tabs")); + } + Ok(()) + } + + fn is_key_token(&self) -> bool { + matches!( + self.current_token, + Token::String(_, _) | Token::Bool(_) | Token::Null + ) + } + + fn key_from_token(&self) -> Option<(String, bool)> { + match &self.current_token { + Token::String(s, was_quoted) => Some((self.format_key(s, *was_quoted), *was_quoted)), + Token::Bool(b) => Some(( + if *b { + KEYWORDS[1].to_string() + } else { + KEYWORDS[2].to_string() + }, + false, + )), + Token::Null => Some((KEYWORDS[0].to_string(), false)), + _ => None, + } + } + + fn find_unexpected_delimiter( + &self, + field: &str, + expected: Option, + ) -> Option { + let expected = expected?; + let delimiters = [Delimiter::Comma, Delimiter::Pipe, Delimiter::Tab]; + + delimiters + .into_iter() + .find(|delim| *delim != expected && field.contains(delim.as_char())) + } + fn parse_value(&mut self) -> ToonResult { self.parse_value_with_depth(0) } @@ -103,7 +187,7 @@ impl<'a> Parser<'a> { if next_char_is_colon { let key = KEYWORDS[0].to_string(); self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { self.advance()?; Ok(Value::Null) @@ -118,7 +202,7 @@ impl<'a> Parser<'a> { KEYWORDS[2].to_string() }; self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { let val = *b; self.advance()?; @@ -130,7 +214,7 @@ impl<'a> Parser<'a> { if next_char_is_colon { let key = i.to_string(); self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { let val = *i; self.advance()?; @@ -142,7 +226,7 @@ impl<'a> Parser<'a> { if next_char_is_colon { let key = n.to_string(); self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { let val = *n; self.advance()?; @@ -158,13 +242,15 @@ impl<'a> Parser<'a> { } } } - Token::String(s, _) => { + Token::String(s, was_quoted) => { + let key_was_quoted = *was_quoted; let first = s.clone(); self.advance()?; match &self.current_token { Token::Colon | Token::LeftBracket => { - self.parse_object_with_initial_key(first, depth) + let key = self.format_key(&first, key_was_quoted); + self.parse_object_with_initial_key(key, key_was_quoted, depth) } _ => { // Strings on new indented lines could be missing colons (keys) or values @@ -189,6 +275,7 @@ impl<'a> Parser<'a> { accumulated.push_str(next); self.advance()?; } + self.validate_unquoted_string(&accumulated, key_was_quoted)?; Ok(Value::String(accumulated)) } } @@ -230,17 +317,9 @@ impl<'a> Parser<'a> { base_indent = Some(current_indent); } - let key = match &self.current_token { - Token::String(s, was_quoted) => { - // Mark quoted keys containing dots with a special prefix - // so path expansion can skip them - if *was_quoted && s.contains('.') { - format!("{QUOTED_KEY_MARKER}{s}") - } else { - s.clone() - } - } - _ => { + let (key, was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { return Err(self .parse_error_with_context(format!( "Expected key, found {:?}", @@ -249,6 +328,7 @@ impl<'a> Parser<'a> { .with_suggestion("Object keys must be strings")); } }; + self.validate_unquoted_key(&key, was_quoted)?; self.advance()?; let value = if matches!(self.current_token, Token::LeftBracket) { @@ -272,7 +352,12 @@ impl<'a> Parser<'a> { Ok(Value::Object(obj)) } - fn parse_object_with_initial_key(&mut self, key: String, depth: usize) -> ToonResult { + fn parse_object_with_initial_key( + &mut self, + key: String, + key_was_quoted: bool, + depth: usize, + ) -> ToonResult { validate_depth(depth, MAX_DEPTH)?; let mut obj = Map::new(); @@ -284,6 +369,8 @@ impl<'a> Parser<'a> { self.validate_indentation(current_indent)?; } + self.validate_unquoted_key(&key, key_was_quoted)?; + if matches!(self.current_token, Token::LeftBracket) { let value = self.parse_array(depth)?; obj.insert(key, value); @@ -335,7 +422,7 @@ impl<'a> Parser<'a> { break; } - if !matches!(self.current_token, Token::String(_, _)) { + if !self.is_key_token() { break; } @@ -365,18 +452,11 @@ impl<'a> Parser<'a> { base_indent = Some(current_indent); } - let key = match &self.current_token { - Token::String(s, was_quoted) => { - // Mark quoted keys containing dots with a special prefix - // so path expansion can skip them - if *was_quoted && s.contains('.') { - format!("{QUOTED_KEY_MARKER}{s}") - } else { - s.clone() - } - } - _ => break, + let (key, was_quoted) = match self.key_from_token() { + Some(key) => key, + None => break, }; + self.validate_unquoted_key(&key, was_quoted)?; self.advance()?; let value = if matches!(self.current_token, Token::LeftBracket) { @@ -416,12 +496,15 @@ impl<'a> Parser<'a> { self.parse_value_with_depth(depth + 1) } else { // Check if there's more content after the current token - let (rest, had_space) = self.scanner.read_rest_of_line_with_space_info(); + let (rest, leading_space) = self.scanner.read_rest_of_line_with_space_info(); let result = if rest.is_empty() { // Single token - convert directly to avoid redundant parsing match &self.current_token { - Token::String(s, _) => Ok(Value::String(s.clone())), + Token::String(s, was_quoted) => { + self.validate_unquoted_string(s, *was_quoted)?; + Ok(Value::String(s.clone())) + } Token::Integer(i) => Ok(serde_json::Number::from(*i).into()), Token::Number(n) => { let val = *n; @@ -461,14 +544,22 @@ impl<'a> Parser<'a> { } // Only add space if there was whitespace in the original input - if had_space { - value_str.push(' '); + if !rest.is_empty() { + value_str.push_str(&leading_space); } value_str.push_str(&rest); let token = self.scanner.parse_value_string(&value_str)?; match token { - Token::String(s, _) => Ok(Value::String(s)), + Token::String(s, was_quoted) => { + if self.options.strict && !was_quoted && value_str.contains('\t') { + return Err(self.parse_error_with_context( + "Unquoted tab characters are not allowed in strict mode", + )); + } + self.validate_unquoted_string(&s, was_quoted)?; + Ok(Value::String(s)) + } Token::Integer(i) => Ok(serde_json::Number::from(i).into()), Token::Number(n) => { if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 { @@ -502,9 +593,7 @@ impl<'a> Parser<'a> { self.parse_array(depth) } - fn parse_array_header( - &mut self, - ) -> ToonResult<(usize, Option, Option>)> { + fn parse_array_header(&mut self) -> ToonResult<(usize, Option, bool)> { if !matches!(self.current_token, Token::LeftBracket) { return Err(self.parse_error_with_context("Expected '['")); } @@ -559,11 +648,6 @@ impl<'a> Parser<'a> { _ => None, }; - // Default to comma if no delimiter specified - let active_delim = detected_delim.or(Some(Delimiter::Comma)); - - self.scanner.set_active_delimiter(active_delim); - if !matches!(self.current_token, Token::RightBracket) { return Err(self.parse_error_with_context(format!( "Expected ']', found {:?}", @@ -572,51 +656,114 @@ impl<'a> Parser<'a> { } self.advance()?; - let fields = if matches!(self.current_token, Token::LeftBrace) { - self.advance()?; - let mut fields = Vec::new(); + let has_fields = matches!(self.current_token, Token::LeftBrace); - loop { - match &self.current_token { - Token::String(s, _) => { - fields.push(s.clone()); - self.advance()?; + Ok((length, detected_delim, has_fields)) + } - if matches!(self.current_token, Token::RightBrace) { - break; - } + fn parse_field_list(&mut self, expected_delim: Option) -> ToonResult> { + if !matches!(self.current_token, Token::LeftBrace) { + return Err(self.parse_error_with_context("Expected '{' for field list")); + } + self.advance()?; - if matches!(self.current_token, Token::Delimiter(_)) { - self.advance()?; - } else { + let mut fields = Vec::new(); + let mut field_list_delim = None; + + loop { + match &self.current_token { + Token::String(s, was_quoted) => { + if self.options.strict { + if let Some(unexpected) = self.find_unexpected_delimiter(s, expected_delim) + { return Err(self.parse_error_with_context(format!( - "Expected delimiter or '}}', found {:?}", - self.current_token + "Field list delimiter {unexpected} does not match expected {}", + expected_delim + .map(|delim| delim.to_string()) + .unwrap_or_else(|| "none".to_string()) ))); } + self.validate_unquoted_key(s, *was_quoted)?; } - Token::RightBrace => break, - _ => { + + fields.push(self.format_key(s, *was_quoted)); + self.advance()?; + + if matches!(self.current_token, Token::RightBrace) { + break; + } + + if let Token::Delimiter(delim) = &self.current_token { + if self.options.strict { + validation::validate_delimiter_consistency( + Some(*delim), + expected_delim, + )?; + } + if field_list_delim.is_none() { + field_list_delim = Some(*delim); + } + self.advance()?; + } else { return Err(self.parse_error_with_context(format!( - "Expected field name, found {:?}", + "Expected delimiter or '}}', found {:?}", self.current_token - ))) + ))); } } - } + Token::Bool(_) | Token::Null => { + let (field, was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { + return Err(self.parse_error_with_context(format!( + "Expected field name, found {:?}", + self.current_token + ))) + } + }; + self.validate_unquoted_key(&field, was_quoted)?; + fields.push(field); + self.advance()?; - self.advance()?; - Some(fields) - } else { - None - }; + if matches!(self.current_token, Token::RightBrace) { + break; + } - if !matches!(self.current_token, Token::Colon) { - return Err(self.parse_error_with_context("Expected ':' after array header")); + if let Token::Delimiter(delim) = &self.current_token { + if self.options.strict { + validation::validate_delimiter_consistency( + Some(*delim), + expected_delim, + )?; + } + if field_list_delim.is_none() { + field_list_delim = Some(*delim); + } + self.advance()?; + } else { + return Err(self.parse_error_with_context(format!( + "Expected delimiter or '}}', found {:?}", + self.current_token + ))); + } + } + Token::RightBrace => break, + _ => { + return Err(self.parse_error_with_context(format!( + "Expected field name, found {:?}", + self.current_token + ))) + } + } } + self.advance()?; + validation::validate_field_list(&fields)?; + if self.options.strict { + validation::validate_delimiter_consistency(field_list_delim, expected_delim)?; + } - Ok((length, detected_delim, fields)) + Ok(fields) } fn parse_array(&mut self, depth: usize) -> ToonResult { @@ -630,20 +777,54 @@ impl<'a> Parser<'a> { ) -> ToonResult { validate_depth(depth, MAX_DEPTH)?; - let (length, _detected_delim, fields) = self.parse_array_header()?; + let (length, detected_delim, has_fields) = self.parse_array_header()?; - if let Some(fields) = fields { - validation::validate_field_list(&fields)?; - self.parse_tabular_array(length, &fields, depth, context) - } else { - // Non-tabular arrays as first field of list items require depth adjustment - // (items at depth +2 relative to hyphen, not the usual +1) - let adjusted_depth = match context { - ArrayParseContext::Normal => depth, - ArrayParseContext::ListItemFirstField => depth + 1, + if let (Some(detected), Some(expected)) = (detected_delim, self.options.delimiter) { + if detected != expected { + return Err(self.parse_error_with_context(format!( + "Detected delimiter {detected} but expected {expected}" + ))); + } + } + + let active_delim = detected_delim + .or(self.options.delimiter) + .or(Some(Delimiter::Comma)); + + let mut pushed = false; + let result = (|| -> ToonResult { + self.push_delimiter(active_delim); + pushed = true; + + let fields = if has_fields { + Some(self.parse_field_list(active_delim)?) + } else { + None }; - self.parse_regular_array(length, adjusted_depth) + + if !matches!(self.current_token, Token::Colon) { + return Err(self.parse_error_with_context("Expected ':' after array header")); + } + self.advance()?; + + if let Some(fields) = fields { + self.parse_tabular_array(length, &fields, depth, context) + } else { + // Non-tabular arrays as first field of list items require depth adjustment + // (items at depth +2 relative to hyphen, not the usual +1) + let adjusted_depth = match context { + ArrayParseContext::Normal => depth, + ArrayParseContext::ListItemFirstField => depth + 1, + }; + self.parse_regular_array(length, adjusted_depth) + } + })(); + + if pushed { + self.pop_delimiter(); } + + result } fn parse_tabular_array( @@ -818,8 +999,8 @@ impl<'a> Parser<'a> { // If something at the same indent level, it might be a new row (error) // unless it's a key-value pair (which belongs to parent) if actual_indent == expected_indent && !matches!(self.current_token, Token::Eof) { - let is_key_value = matches!(self.current_token, Token::String(_, _)) - && matches!(self.scanner.peek(), Some(':')); + let is_key_value = + self.is_key_token() && matches!(self.scanner.peek(), Some(':')); if !is_key_value { return Err(self.parse_error_with_context(format!( @@ -874,8 +1055,17 @@ impl<'a> Parser<'a> { Value::Object(Map::new()) } else if matches!(self.current_token, Token::LeftBracket) { self.parse_array(depth + 1)? - } else if let Token::String(s, _) = &self.current_token { - let key = s.clone(); + } else if self.is_key_token() { + let (key, key_was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { + return Err(self.parse_error_with_context(format!( + "Expected key, found {:?}", + self.current_token + ))); + } + }; + self.validate_unquoted_key(&key, key_was_quoted)?; self.advance()?; if matches!(self.current_token, Token::Colon | Token::LeftBracket) { @@ -921,7 +1111,7 @@ impl<'a> Parser<'a> { } true } - } else if matches!(self.current_token, Token::String(_, _)) { + } else if self.is_key_token() { // When already positioned at a field key, check its indent let current_indent = self.scanner.get_last_line_indent(); current_indent == field_indent @@ -947,10 +1137,12 @@ impl<'a> Parser<'a> { break; } - let field_key = match &self.current_token { - Token::String(s, _) => s.clone(), - _ => break, - }; + let (field_key, field_key_was_quoted) = + match self.key_from_token() { + Some(key) => key, + None => break, + }; + self.validate_unquoted_key(&field_key, field_key_was_quoted)?; self.advance()?; let field_value = @@ -1109,12 +1301,14 @@ impl<'a> Parser<'a> { .into()) } } - Token::String(s, _) => { + Token::String(s, was_quoted) => { // Tabular fields can have multiple string tokens joined with spaces + self.validate_unquoted_string(s, *was_quoted)?; let mut accumulated = s.clone(); self.advance()?; - while let Token::String(next, _) = &self.current_token { + while let Token::String(next, next_was_quoted) = &self.current_token { + self.validate_unquoted_string(next, *next_was_quoted)?; if !accumulated.is_empty() { accumulated.push(' '); } @@ -1159,7 +1353,8 @@ impl<'a> Parser<'a> { .into()) } } - Token::String(s, _) => { + Token::String(s, was_quoted) => { + self.validate_unquoted_string(s, *was_quoted)?; let val = s.clone(); self.advance()?; Ok(Value::String(val)) diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 9ca7615..bf71f13 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -27,6 +27,7 @@ pub struct Scanner { column: usize, active_delimiter: Option, last_line_indent: usize, + coerce_types: bool, } impl Scanner { @@ -39,6 +40,7 @@ impl Scanner { column: 1, active_delimiter: None, last_line_indent: 0, + coerce_types: true, } } @@ -47,6 +49,10 @@ impl Scanner { self.active_delimiter = delimiter; } + pub fn set_coerce_types(&mut self, coerce_types: bool) { + self.coerce_types = coerce_types; + } + /// Get the current position (line, column). pub fn current_position(&self) -> (usize, usize) { (self.line, self.column) @@ -293,6 +299,10 @@ impl Scanner { value.trim_end().to_string() }; + if !self.coerce_types { + return Ok(Token::String(value, false)); + } + match value.as_str() { "null" => Ok(Token::Null), "true" => Ok(Token::Bool(true)), @@ -326,6 +336,10 @@ impl Scanner { } fn parse_number(&self, s: &str) -> ToonResult { + if !self.coerce_types { + return Ok(Token::String(s.to_string(), false)); + } + // Number followed immediately by other chars like "0(f)" should be a string if let Some(next_ch) = self.peek() { if next_ch != ' ' @@ -354,6 +368,13 @@ impl Scanner { } } } + if s.starts_with("-0") && s.len() > 2 { + if let Some(third_char) = s.chars().nth(2) { + if third_char.is_ascii_digit() { + return Ok(Token::String(s.to_string(), false)); + } + } + } if s.contains('.') || s.contains('e') || s.contains('E') { if let Ok(f) = s.parse::() { @@ -369,11 +390,14 @@ impl Scanner { } /// Read the rest of the current line (until newline or EOF). - /// Returns the content with a flag indicating if it started with - /// whitespace. - pub fn read_rest_of_line_with_space_info(&mut self) -> (String, bool) { - let had_leading_space = matches!(self.peek(), Some(' ')); - self.skip_whitespace(); + /// Returns the content and any leading spaces between the current token + /// and the rest of the line. + pub fn read_rest_of_line_with_space_info(&mut self) -> (String, String) { + let mut leading_space = String::new(); + while matches!(self.peek(), Some(' ')) { + leading_space.push(' '); + self.advance(); + } let mut result = String::new(); while let Some(ch) = self.peek() { @@ -384,7 +408,7 @@ impl Scanner { self.advance(); } - (result.trim_end().to_string(), had_leading_space) + (result.trim_end().to_string(), leading_space) } /// Read the rest of the current line (until newline or EOF). @@ -448,6 +472,10 @@ impl Scanner { )); } + if !self.coerce_types { + return Ok(Token::String(trimmed.to_string(), false)); + } + match trimmed { "true" => return Ok(Token::Bool(true)), "false" => return Ok(Token::Bool(false)), @@ -455,13 +483,8 @@ impl Scanner { _ => {} } - if trimmed.starts_with('-') - || trimmed - .chars() - .next() - .is_some_and(|c| c.is_ascii_digit()) - { - // Leading zeros like "05" are strings + if trimmed.starts_with('-') || trimmed.chars().next().is_some_and(|c| c.is_ascii_digit()) { + // Leading zeros like "05" or "-05" are strings if trimmed.starts_with('0') && trimmed.len() > 1 { if let Some(second_char) = trimmed.chars().nth(1) { if second_char.is_ascii_digit() { @@ -469,6 +492,13 @@ impl Scanner { } } } + if trimmed.starts_with("-0") && trimmed.len() > 2 { + if let Some(third_char) = trimmed.chars().nth(2) { + if third_char.is_ascii_digit() { + return Ok(Token::String(trimmed.to_string(), false)); + } + } + } if trimmed.contains('.') || trimmed.contains('e') || trimmed.contains('E') { if let Ok(f) = trimmed.parse::() { @@ -595,24 +625,24 @@ mod tests { #[test] fn test_read_rest_of_line_with_space_info() { let mut scanner = Scanner::new(" world"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "world"); - assert!(had_space); + assert_eq!(leading_space, " "); let mut scanner = Scanner::new("world"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "world"); - assert!(!had_space); + assert!(leading_space.is_empty()); let mut scanner = Scanner::new("(hello)"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "(hello)"); - assert!(!had_space); + assert!(leading_space.is_empty()); let mut scanner = Scanner::new(""); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, ""); - assert!(!had_space); + assert!(leading_space.is_empty()); } #[test] diff --git a/src/types/folding.rs b/src/types/folding.rs index 2c7d272..ce968f1 100644 --- a/src/types/folding.rs +++ b/src/types/folding.rs @@ -31,12 +31,12 @@ pub fn is_identifier_segment(s: &str) -> bool { None => return false, }; - if !first.is_alphabetic() && first != '_' { + if !first.is_ascii_alphabetic() && first != '_' { return false; } // Remaining characters: letters, digits, or underscore (NO dots) - chars.all(|c| c.is_alphanumeric() || c == '_') + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') } #[cfg(test)] diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 1b300cb..f8ad2f1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,6 +6,7 @@ pub mod validation; use indexmap::IndexMap; pub use literal::{is_keyword, is_literal_like, is_numeric_like, is_structural_char}; pub use number::format_canonical_number; +pub(crate) use string::is_valid_unquoted_key_decode; pub use string::{ escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, }; diff --git a/src/utils/string.rs b/src/utils/string.rs index 3622990..fc0b2a9 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -89,9 +89,7 @@ pub fn unescape_string(s: &str) -> Result { Ok(result) } -/// Check if a key can be written without quotes (alphanumeric, underscore, -/// dot). -pub fn is_valid_unquoted_key(key: &str) -> bool { +fn is_valid_unquoted_key_internal(key: &str, allow_hyphen: bool) -> bool { if key.is_empty() { return false; } @@ -103,11 +101,21 @@ pub fn is_valid_unquoted_key(key: &str) -> bool { return false; }; - if !first.is_alphabetic() && first != '_' { + if !first.is_ascii_alphabetic() && first != '_' { return false; } - chars.all(|c| c.is_alphanumeric() || c == '_' || c == '.') + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.' || (allow_hyphen && c == '-')) +} + +/// Check if a key can be written without quotes (alphanumeric, underscore, +/// dot). +pub fn is_valid_unquoted_key(key: &str) -> bool { + is_valid_unquoted_key_internal(key, false) +} + +pub(crate) fn is_valid_unquoted_key_decode(key: &str) -> bool { + is_valid_unquoted_key_internal(key, true) } /// Determine if a string needs quoting based on content and delimiter. @@ -295,5 +303,6 @@ mod tests { assert!(is_valid_unquoted_key("key.")); assert!(!is_valid_unquoted_key("key[value]")); assert!(!is_valid_unquoted_key("key{value}")); + assert!(is_valid_unquoted_key_decode("key-value")); } } From bde58a82b261e8c7848204c172aa52303342bce9 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 14:02:31 +0000 Subject: [PATCH 11/24] test: add decoder edge cases --- tests/delimiters.rs | 10 +++---- tests/errors.rs | 8 +++--- tests/spec_edge_cases.rs | 59 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 tests/spec_edge_cases.rs diff --git a/tests/delimiters.rs b/tests/delimiters.rs index 7e52a67..dd0ef38 100644 --- a/tests/delimiters.rs +++ b/tests/delimiters.rs @@ -45,7 +45,7 @@ fn test_non_active_delimiters_in_tabular_arrays() { // data Per TOON spec §11: "non-active delimiters MUST NOT cause splits" // Test 1: Pipe character in value when comma is active delimiter (default) - let data = r#"item-list[1]{a,b}: + let data = r#""item-list"[1]{a,b}: ":",| "#; let decoded: Value = decode_default(data).unwrap(); @@ -53,7 +53,7 @@ fn test_non_active_delimiters_in_tabular_arrays() { assert_eq!(decoded["item-list"][0]["b"], "|"); // Test 2: Both values quoted - let data = r#"item-list[1]{a,b}: + let data = r#""item-list"[1]{a,b}: ":","|" "#; let decoded: Value = decode_default(data).unwrap(); @@ -61,13 +61,13 @@ fn test_non_active_delimiters_in_tabular_arrays() { assert_eq!(decoded["item-list"][0]["b"], "|"); // Test 3: Tab character in value when comma is active - let data = "item-list[1]{a,b}:\n \":\",\t\n"; + let data = "\"item-list\"[1]{a,b}:\n \":\",\"\\t\"\n"; let decoded: Value = decode_default(data).unwrap(); assert_eq!(decoded["item-list"][0]["a"], ":"); assert_eq!(decoded["item-list"][0]["b"], "\t"); // Test 4: Comma in value when pipe is active delimiter - should quote the comma - let data = r#"item-list[1|]{a|b}: + let data = r#""item-list"[1|]{a|b}: ":"|"," "#; let decoded: Value = decode_default(data).unwrap(); @@ -97,7 +97,7 @@ fn test_non_active_delimiters_in_inline_arrays() { fn test_delimiter_mismatch_error() { // Per TOON spec §6: delimiter in brackets must match delimiter in braces // This should error: pipe in brackets, comma in braces - let data = r#"item-list[1|]{a,b}: + let data = r#""item-list"[1|]{a,b}: ":",| "#; let result: Result = decode_default(data); diff --git a/tests/errors.rs b/tests/errors.rs index a100332..45fa38b 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -271,12 +271,12 @@ fn test_no_coercion_preserves_strings() { assert_eq!(result["value"], json!("true")); let result = decode::("value: 123", &opts).unwrap(); - assert!(result["value"].is_number()); - assert_eq!(result["value"], json!(123)); + assert!(result["value"].is_string()); + assert_eq!(result["value"], json!("123")); let result = decode::("value: true", &opts).unwrap(); - assert!(result["value"].is_boolean()); - assert_eq!(result["value"], json!(true)); + assert!(result["value"].is_string()); + assert_eq!(result["value"], json!("true")); } #[test] diff --git a/tests/spec_edge_cases.rs b/tests/spec_edge_cases.rs new file mode 100644 index 0000000..9fb5a8a --- /dev/null +++ b/tests/spec_edge_cases.rs @@ -0,0 +1,59 @@ +//! Spec compliance edge cases + +use serde_json::json; +use toon_format::types::PathExpansionMode; +use toon_format::{decode, decode_default, DecodeOptions}; + +#[test] +fn test_keyword_keys_allowed() { + let input = "true: 1\nfalse: 2\nnull: 3"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"true": 1, "false": 2, "null": 3})); +} + +#[test] +fn test_nested_array_delimiter_scoping() { + let input = "outer[2|]: [2]: a,b | [2]: c,d"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"outer": [["a", "b"], ["c", "d"]]})); +} + +#[test] +fn test_quoted_dotted_field_not_expanded() { + let input = "rows[1]{\"a.b\"}:\n 1"; + let opts = DecodeOptions::new().with_expand_paths(PathExpansionMode::Safe); + let result: serde_json::Value = decode(input, &opts).unwrap(); + assert_eq!(result, json!({"rows": [{"a.b": 1}]})); +} + +#[test] +fn test_negative_leading_zero_string() { + let input = "val: -05"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"val": "-05"})); +} + +#[test] +fn test_unquoted_tab_rejected_in_strict() { + let input = "val: a\tb"; + let result: Result = decode_default(input); + assert!(result.is_err()); +} + +#[test] +fn test_multiple_spaces_preserved() { + let input = "msg: hello world"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"msg": "hello world"})); +} + +#[test] +fn test_coerce_types_toggle() { + let input = "value: 123\nflag: true\nnone: null"; + let opts = DecodeOptions::new().with_coerce_types(false); + let result: serde_json::Value = decode(input, &opts).unwrap(); + assert_eq!( + result, + json!({"value": "123", "flag": "true", "none": "null"}) + ); +} From 085e4666596dea3dcc4bb6b142dd53479380a599 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 15:34:51 +0000 Subject: [PATCH 12/24] fix(strict): enforce strict validations and relax non-strict --- src/cli/main.rs | 4 - src/decode/parser.rs | 759 ++++++++++++++++++++++++++------------- src/decode/scanner.rs | 88 +++-- src/decode/validation.rs | 37 +- src/types/value.rs | 5 +- src/utils/mod.rs | 1 - src/utils/string.rs | 5 - src/utils/validation.rs | 6 +- tests/strict_mode.rs | 67 ++++ 9 files changed, 662 insertions(+), 310 deletions(-) create mode 100644 tests/strict_mode.rs diff --git a/src/cli/main.rs b/src/cli/main.rs index 311fb00..d734f53 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -167,10 +167,6 @@ fn run_encode(cli: &Cli, input: &str) -> Result<()> { write_output(cli.output.clone(), &toon_str)?; - if cli.output.is_none() && !toon_str.ends_with('\n') { - io::stdout().write_all(b"\n")?; - } - if cli.stats { let json_bytes = input.len(); let toon_bytes = toon_str.len(); diff --git a/src/decode/parser.rs b/src/decode/parser.rs index a682a9d..38df222 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -6,8 +6,8 @@ use crate::{ scanner::{Scanner, Token}, validation, }, - types::{DecodeOptions, Delimiter, ErrorContext, ToonError, ToonResult}, - utils::{is_valid_unquoted_key_decode, validation::validate_depth}, + types::{DecodeOptions, Delimiter, ErrorContext, PathExpansionMode, ToonError, ToonResult}, + utils::{is_valid_unquoted_key, validation::validate_depth}, }; /// Context for parsing arrays to determine correct indentation depth. @@ -43,6 +43,7 @@ impl<'a> Parser<'a> { let chosen_delim = options.delimiter; scanner.set_active_delimiter(chosen_delim); scanner.set_coerce_types(options.coerce_types); + scanner.configure_indentation(options.strict, options.indent.get_spaces()); let current_token = scanner.scan_token()?; Ok(Self { @@ -117,10 +118,16 @@ impl<'a> Parser<'a> { } fn validate_unquoted_key(&self, key: &str, was_quoted: bool) -> ToonResult<()> { - if self.options.strict && !was_quoted && !is_valid_unquoted_key_decode(key) { - return Err(self - .parse_error_with_context(format!("Invalid unquoted key: '{key}'")) - .with_suggestion("Quote the key to use special characters")); + if self.options.strict && !was_quoted { + if self.options.expand_paths != PathExpansionMode::Off && key.contains('.') { + return Ok(()); + } + + if !is_valid_unquoted_key(key) { + return Err(self + .parse_error_with_context(format!("Invalid unquoted key: '{key}'")) + .with_suggestion("Quote the key to use special characters")); + } } Ok(()) } @@ -302,7 +309,7 @@ impl<'a> Parser<'a> { break; } - let current_indent = self.scanner.get_last_line_indent(); + let current_indent = self.normalize_indent(self.scanner.get_last_line_indent()); if self.options.strict { self.validate_indentation(current_indent)?; @@ -365,7 +372,7 @@ impl<'a> Parser<'a> { // Validate indentation for the initial key if in strict mode if self.options.strict { - let current_indent = self.scanner.get_last_line_indent(); + let current_indent = self.normalize_indent(self.scanner.get_last_line_indent()); self.validate_indentation(current_indent)?; } @@ -402,7 +409,7 @@ impl<'a> Parser<'a> { continue; } - let next_indent = self.scanner.get_last_line_indent(); + let next_indent = self.normalize_indent(self.scanner.get_last_line_indent()); // Check if the next line is at the right indentation level let should_continue = if let Some(expected) = base_indent { @@ -430,7 +437,7 @@ impl<'a> Parser<'a> { break; } - let current_indent = self.scanner.get_last_line_indent(); + let current_indent = self.normalize_indent(self.scanner.get_last_line_indent()); if let Some(expected) = base_indent { if current_indent != expected { @@ -481,7 +488,7 @@ impl<'a> Parser<'a> { if matches!(self.current_token, Token::Newline | Token::Eof) { let has_children = if matches!(self.current_token, Token::Newline) { let current_depth_indent = self.options.indent.get_spaces() * (depth + 1); - let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self.normalize_indent(self.scanner.count_leading_spaces()); next_indent >= current_depth_indent } else { false @@ -524,7 +531,17 @@ impl<'a> Parser<'a> { } } else { // Multi-token value - reconstruct and re-parse as complete string - let mut value_str = String::new(); + let token_len = match &self.current_token { + Token::String(s, was_quoted) => s.len() + if *was_quoted { 2 } else { 0 }, + Token::Integer(_) => 20, + Token::Number(_) => 32, + Token::Bool(true) => 4, + Token::Bool(false) => 5, + Token::Null => 4, + _ => 0, + }; + let mut value_str = + String::with_capacity(token_len + leading_space.len() + rest.len()); match &self.current_token { Token::String(s, true) => { @@ -601,27 +618,40 @@ impl<'a> Parser<'a> { // Parse array length (plain integer only) // Supports formats: [N], [N|], [N\t] (no # marker) - let length = if let Token::Integer(n) = &self.current_token { - *n as usize - } else if let Token::String(s, _) = &self.current_token { - // Check if string starts with # - this marker is not supported - if s.starts_with('#') { - return Err(self - .parse_error_with_context( - "Length marker '#' is not supported. Use [N] format instead of [#N]", - ) - .with_suggestion("Remove the '#' prefix from the array length")); + let length = match &self.current_token { + Token::Integer(n) => { + validation::validate_array_length_non_negative(*n)?; + *n as usize } + Token::Number(_) => { + return Err(self.parse_error_with_context("Array length must be an integer")); + } + Token::String(s, _) => { + // Check if string starts with # - this marker is not supported + if s.starts_with('#') { + return Err(self + .parse_error_with_context( + "Length marker '#' is not supported. Use [N] format instead of [#N]", + ) + .with_suggestion("Remove the '#' prefix from the array length")); + } - // Plain string that's a number: "3" - s.parse::().map_err(|_| { - self.parse_error_with_context(format!("Expected array length, found: {s}")) - })? - } else { - return Err(self.parse_error_with_context(format!( - "Expected array length, found {:?}", - self.current_token - ))); + if s.contains('.') || s.contains('e') || s.contains('E') { + return Err(self.parse_error_with_context("Array length must be an integer")); + } + + let parsed = s.parse::().map_err(|_| { + self.parse_error_with_context(format!("Expected array length, found: {s}")) + })?; + validation::validate_array_length_non_negative(parsed)?; + parsed as usize + } + _ => { + return Err(self.parse_error_with_context(format!( + "Expected array length, found {:?}", + self.current_token + ))); + } }; self.advance()?; @@ -834,7 +864,7 @@ impl<'a> Parser<'a> { depth: usize, context: ArrayParseContext, ) -> ToonResult { - let mut rows = Vec::new(); + let mut rows = Vec::with_capacity(length); if !matches!(self.current_token, Token::Newline) { return Err(self @@ -843,7 +873,17 @@ impl<'a> Parser<'a> { } self.skip_newlines()?; - for row_index in 0..length { + // Tabular arrays as first field of list-item objects require rows at depth +2 + // (relative to hyphen), while normal tabular arrays use depth +1 + let row_depth_offset = match context { + ArrayParseContext::Normal => 1, + ArrayParseContext::ListItemFirstField => 2, + }; + let indent_size = self.options.indent.get_spaces(); + let expected_indent = indent_size * (depth + row_depth_offset); + + let mut row_index = 0; + loop { if matches!(self.current_token, Token::Eof) { if self.options.strict { return Err(self.parse_error_with_context(format!( @@ -855,15 +895,7 @@ impl<'a> Parser<'a> { break; } - let current_indent = self.scanner.get_last_line_indent(); - - // Tabular arrays as first field of list-item objects require rows at depth +2 - // (relative to hyphen), while normal tabular arrays use depth +1 - let row_depth_offset = match context { - ArrayParseContext::Normal => 1, - ArrayParseContext::ListItemFirstField => 2, - }; - let expected_indent = self.options.indent.get_spaces() * (depth + row_depth_offset); + let current_indent = self.normalize_indent(self.scanner.get_last_line_indent()); if self.options.strict { self.validate_indentation(current_indent)?; @@ -874,9 +906,14 @@ impl<'a> Parser<'a> { found {current_indent}" ))); } + } else { + let is_key_value = self.is_key_token() && matches!(self.scanner.peek(), Some(':')); + if current_indent != expected_indent || is_key_value { + break; + } } - let mut row = Map::new(); + let mut row = Map::with_capacity(fields.len()); for (field_index, field) in fields.iter().enumerate() { // Skip delimiter before each field except the first @@ -958,6 +995,7 @@ impl<'a> Parser<'a> { } rows.push(Value::Object(row)); + row_index += 1; if matches!(self.current_token, Token::Eof) { break; @@ -974,60 +1012,73 @@ impl<'a> Parser<'a> { } else { return Err(self.parse_error_with_context(format!( "Expected newline after tabular row {}", - row_index + 1 + row_index ))); } } - if row_index + 1 < length { - self.advance()?; - if self.options.strict && matches!(self.current_token, Token::Newline) { - return Err(self.parse_error_with_context( - "Blank lines are not allowed inside tabular arrays in strict mode", - )); - } + if self.options.strict { + if row_index < length { + self.advance()?; + if matches!(self.current_token, Token::Newline) { + return Err(self.parse_error_with_context( + "Blank lines are not allowed inside tabular arrays in strict mode", + )); + } - self.skip_newlines()?; - } else if matches!(self.current_token, Token::Newline) { - // After the last row, check if there are extra rows - self.advance()?; - self.skip_newlines()?; + self.skip_newlines()?; + } else if matches!(self.current_token, Token::Newline) { + // After the last row, check if there are extra rows + self.advance()?; + self.skip_newlines()?; - let expected_indent = self.options.indent.get_spaces() * (depth + 1); - let actual_indent = self.scanner.get_last_line_indent(); + let actual_indent = self.normalize_indent(self.scanner.get_last_line_indent()); - // If something at the same indent level, it might be a new row (error) - // unless it's a key-value pair (which belongs to parent) - if actual_indent == expected_indent && !matches!(self.current_token, Token::Eof) { - let is_key_value = - self.is_key_token() && matches!(self.scanner.peek(), Some(':')); + // If something at the same indent level, it might be a new row (error) + // unless it's a key-value pair (which belongs to parent) + if actual_indent == expected_indent && !matches!(self.current_token, Token::Eof) + { + let is_key_value = + self.is_key_token() && matches!(self.scanner.peek(), Some(':')); - if !is_key_value { - return Err(self.parse_error_with_context(format!( - "Array length mismatch: expected {length} rows, but more rows found", - ))); + if !is_key_value { + return Err(self.parse_error_with_context(format!( + "Array length mismatch: expected {length} rows, but more rows found", + ))); + } } } + + if row_index >= length { + break; + } + } else if matches!(self.current_token, Token::Newline) { + self.advance()?; + self.skip_newlines()?; } } - validation::validate_array_length(length, rows.len())?; + if self.options.strict { + validation::validate_array_length(length, rows.len())?; + } Ok(Value::Array(rows)) } fn parse_regular_array(&mut self, length: usize, depth: usize) -> ToonResult { - let mut items = Vec::new(); + let mut items = Vec::with_capacity(length); + let indent_size = self.options.indent.get_spaces(); match &self.current_token { Token::Newline => { self.skip_newlines()?; - let expected_indent = self.options.indent.get_spaces() * (depth + 1); + let expected_indent = indent_size * (depth + 1); - for i in 0..length { - let current_indent = self.scanner.get_last_line_indent(); - if self.options.strict { + if self.options.strict { + for i in 0..length { + let current_indent = + self.normalize_indent(self.scanner.get_last_line_indent()); self.validate_indentation(current_indent)?; if current_indent != expected_indent { @@ -1036,238 +1087,445 @@ impl<'a> Parser<'a> { spaces, found {current_indent}" ))); } - } - if !matches!(self.current_token, Token::Dash) { - return Err(self - .parse_error_with_context(format!( - "Expected '-' for list item, found {:?}", - self.current_token - )) - .with_suggestion(format!( - "List arrays need '-' prefix for each item (item {} of {})", - i + 1, - length - ))); - } - self.advance()?; - - let value = if matches!(self.current_token, Token::Newline | Token::Eof) { - Value::Object(Map::new()) - } else if matches!(self.current_token, Token::LeftBracket) { - self.parse_array(depth + 1)? - } else if self.is_key_token() { - let (key, key_was_quoted) = match self.key_from_token() { - Some(key) => key, - None => { - return Err(self.parse_error_with_context(format!( - "Expected key, found {:?}", + if !matches!(self.current_token, Token::Dash) { + return Err(self + .parse_error_with_context(format!( + "Expected '-' for list item, found {:?}", self.current_token + )) + .with_suggestion(format!( + "List arrays need '-' prefix for each item (item {} of {})", + i + 1, + length ))); - } - }; - self.validate_unquoted_key(&key, key_was_quoted)?; + } self.advance()?; - if matches!(self.current_token, Token::Colon | Token::LeftBracket) { - // This is an object: key followed by colon or array bracket - // First field of list-item object may be an array requiring special - // indentation - let first_value = if matches!(self.current_token, Token::LeftBracket) { - // Array directly after key (e.g., "- key[N]:") - // Use ListItemFirstField context to apply correct indentation - self.parse_array_with_context( - depth + 1, - ArrayParseContext::ListItemFirstField, - )? - } else { - self.advance()?; - // Handle nested arrays: "key: [2]: ..." - if matches!(self.current_token, Token::LeftBracket) { - // Array after colon - not directly on hyphen line, use normal - // context - self.parse_array(depth + 2)? - } else { - self.parse_field_value(depth + 2)? + let value = if matches!(self.current_token, Token::Newline | Token::Eof) { + Value::Object(Map::new()) + } else if matches!(self.current_token, Token::LeftBracket) { + self.parse_array(depth + 1)? + } else if self.is_key_token() { + let (key, key_was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { + return Err(self.parse_error_with_context(format!( + "Expected key, found {:?}", + self.current_token + ))); } }; + self.validate_unquoted_key(&key, key_was_quoted)?; + self.advance()?; - let mut obj = Map::new(); - obj.insert(key, first_value); + if matches!(self.current_token, Token::Colon | Token::LeftBracket) { + // This is an object: key followed by colon or array bracket + // First field of list-item object may be an array requiring special + // indentation + let first_value = + if matches!(self.current_token, Token::LeftBracket) { + // Array directly after key (e.g., "- key[N]:") + // Use ListItemFirstField context to apply correct indentation + self.parse_array_with_context( + depth + 1, + ArrayParseContext::ListItemFirstField, + )? + } else { + self.advance()?; + // Handle nested arrays: "key: [2]: ..." + if matches!(self.current_token, Token::LeftBracket) { + // Array after colon - not directly on hyphen line, use normal + // context + self.parse_array(depth + 2)? + } else { + self.parse_field_value(depth + 2)? + } + }; - let field_indent = self.options.indent.get_spaces() * (depth + 2); + let mut obj = Map::new(); + obj.insert(key, first_value); - // Check if there are more fields at the same indentation level - let should_parse_more_fields = - if matches!(self.current_token, Token::Newline) { - let next_indent = self.scanner.count_leading_spaces(); + let field_indent = indent_size * (depth + 2); - if next_indent < field_indent { - false - } else { - self.advance()?; + // Check if there are more fields at the same indentation level + let should_parse_more_fields = + if matches!(self.current_token, Token::Newline) { + let next_indent = self + .normalize_indent(self.scanner.count_leading_spaces()); - if !self.options.strict { - self.skip_newlines()?; + if next_indent < field_indent { + false + } else { + self.advance()?; + + if !self.options.strict { + self.skip_newlines()?; + } + true } - true - } - } else if self.is_key_token() { - // When already positioned at a field key, check its indent - let current_indent = self.scanner.get_last_line_indent(); - current_indent == field_indent - } else { - false - }; - - // Parse additional fields if they're at the right indentation - if should_parse_more_fields { - while !matches!(self.current_token, Token::Eof) { - let current_indent = self.scanner.get_last_line_indent(); - - if current_indent < field_indent { - break; - } + } else if self.is_key_token() { + // When already positioned at a field key, check its indent + let current_indent = self + .normalize_indent(self.scanner.get_last_line_indent()); + current_indent == field_indent + } else { + false + }; - if current_indent != field_indent && self.options.strict { - break; - } + // Parse additional fields if they're at the right indentation + if should_parse_more_fields { + while !matches!(self.current_token, Token::Eof) { + let current_indent = self + .normalize_indent(self.scanner.get_last_line_indent()); - // Stop if we hit the next list item - if matches!(self.current_token, Token::Dash) { - break; - } + if current_indent != field_indent { + break; + } - let (field_key, field_key_was_quoted) = - match self.key_from_token() { - Some(key) => key, - None => break, - }; - self.validate_unquoted_key(&field_key, field_key_was_quoted)?; - self.advance()?; + // Stop if we hit the next list item + if matches!(self.current_token, Token::Dash) { + break; + } - let field_value = - if matches!(self.current_token, Token::LeftBracket) { - self.parse_array(depth + 2)? - } else if matches!(self.current_token, Token::Colon) { - self.advance()?; + let (field_key, field_key_was_quoted) = + match self.key_from_token() { + Some(key) => key, + None => break, + }; + self.validate_unquoted_key( + &field_key, + field_key_was_quoted, + )?; + self.advance()?; + + let field_value = if matches!(self.current_token, Token::LeftBracket) { self.parse_array(depth + 2)? + } else if matches!(self.current_token, Token::Colon) { + self.advance()?; + if matches!(self.current_token, Token::LeftBracket) + { + self.parse_array(depth + 2)? + } else { + self.parse_field_value(depth + 2)? + } } else { - self.parse_field_value(depth + 2)? + break; + }; + + obj.insert(field_key, field_value); + + if matches!(self.current_token, Token::Newline) { + let next_indent = self.normalize_indent( + self.scanner.count_leading_spaces(), + ); + if next_indent < field_indent { + break; + } + self.advance()?; + if !self.options.strict { + self.skip_newlines()?; } } else { break; - }; + } + } + } + + Value::Object(obj) + } else if matches!(self.current_token, Token::LeftBracket) { + // Array as object value: "key[2]: ..." + let array_value = self.parse_array(depth + 1)?; + let mut obj = Map::new(); + obj.insert(key, array_value); + Value::Object(obj) + } else { + // Plain string value + Value::String(key) + } + } else { + self.parse_primitive()? + }; + + items.push(value); + + if items.len() < length { + if matches!(self.current_token, Token::Newline) { + self.advance()?; + + if self.options.strict + && matches!(self.current_token, Token::Newline) + { + return Err(self.parse_error_with_context( + "Blank lines are not allowed inside list arrays in strict mode", + )); + } + + self.skip_newlines()?; + } else if !matches!(self.current_token, Token::Dash) { + return Err(self.parse_error_with_context(format!( + "Expected newline or next list item after list item {}", + i + 1 + ))); + } + } else if matches!(self.current_token, Token::Newline) { + // After the last item, check for extra items + self.advance()?; + self.skip_newlines()?; + + let list_indent = indent_size * (depth + 1); + let actual_indent = + self.normalize_indent(self.scanner.get_last_line_indent()); + // If we see another dash at the same indent, there are too many items + if actual_indent == list_indent + && matches!(self.current_token, Token::Dash) + { + return Err(self.parse_error_with_context(format!( + "Array length mismatch: expected {length} items, but more items \ + found", + ))); + } + } + } + } else { + loop { + if matches!(self.current_token, Token::Eof) { + break; + } + + let current_indent = + self.normalize_indent(self.scanner.get_last_line_indent()); + if current_indent != expected_indent { + break; + } + + if !matches!(self.current_token, Token::Dash) { + break; + } + self.advance()?; + + let value = if matches!(self.current_token, Token::Newline | Token::Eof) { + Value::Object(Map::new()) + } else if matches!(self.current_token, Token::LeftBracket) { + self.parse_array(depth + 1)? + } else if self.is_key_token() { + let (key, key_was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { + return Err(self.parse_error_with_context(format!( + "Expected key, found {:?}", + self.current_token + ))); + } + }; + self.validate_unquoted_key(&key, key_was_quoted)?; + self.advance()?; + + if matches!(self.current_token, Token::Colon | Token::LeftBracket) { + let first_value = + if matches!(self.current_token, Token::LeftBracket) { + self.parse_array_with_context( + depth + 1, + ArrayParseContext::ListItemFirstField, + )? + } else { + self.advance()?; + if matches!(self.current_token, Token::LeftBracket) { + self.parse_array(depth + 2)? + } else { + self.parse_field_value(depth + 2)? + } + }; - obj.insert(field_key, field_value); + let mut obj = Map::new(); + obj.insert(key, first_value); + let field_indent = indent_size * (depth + 2); + + let should_parse_more_fields = if matches!(self.current_token, Token::Newline) { - let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self + .normalize_indent(self.scanner.count_leading_spaces()); + if next_indent < field_indent { + false + } else { + self.advance()?; + self.skip_newlines()?; + true + } + } else if self.is_key_token() { + let current_indent = self + .normalize_indent(self.scanner.get_last_line_indent()); + current_indent == field_indent + } else { + false + }; + + if should_parse_more_fields { + while !matches!(self.current_token, Token::Eof) { + let current_indent = self + .normalize_indent(self.scanner.get_last_line_indent()); + if current_indent != field_indent { + break; + } + + if matches!(self.current_token, Token::Dash) { break; } + + let (field_key, field_key_was_quoted) = + match self.key_from_token() { + Some(key) => key, + None => break, + }; + self.validate_unquoted_key( + &field_key, + field_key_was_quoted, + )?; self.advance()?; - if !self.options.strict { + + let field_value = + if matches!(self.current_token, Token::LeftBracket) { + self.parse_array(depth + 2)? + } else if matches!(self.current_token, Token::Colon) { + self.advance()?; + if matches!(self.current_token, Token::LeftBracket) + { + self.parse_array(depth + 2)? + } else { + self.parse_field_value(depth + 2)? + } + } else { + break; + }; + + obj.insert(field_key, field_value); + + if matches!(self.current_token, Token::Newline) { + let next_indent = self.normalize_indent( + self.scanner.count_leading_spaces(), + ); + if next_indent < field_indent { + break; + } + self.advance()?; self.skip_newlines()?; + } else { + break; } - } else { - break; } } - } - Value::Object(obj) - } else if matches!(self.current_token, Token::LeftBracket) { - // Array as object value: "key[2]: ..." - let array_value = self.parse_array(depth + 1)?; - let mut obj = Map::new(); - obj.insert(key, array_value); - Value::Object(obj) + Value::Object(obj) + } else if matches!(self.current_token, Token::LeftBracket) { + let array_value = self.parse_array(depth + 1)?; + let mut obj = Map::new(); + obj.insert(key, array_value); + Value::Object(obj) + } else { + Value::String(key) + } } else { - // Plain string value - Value::String(key) - } - } else { - self.parse_primitive()? - }; + self.parse_primitive()? + }; - items.push(value); + items.push(value); - if items.len() < length { if matches!(self.current_token, Token::Newline) { self.advance()?; - - if self.options.strict && matches!(self.current_token, Token::Newline) { - return Err(self.parse_error_with_context( - "Blank lines are not allowed inside list arrays in strict mode", - )); - } - self.skip_newlines()?; + } else if matches!(self.current_token, Token::Eof) { + break; } else if !matches!(self.current_token, Token::Dash) { return Err(self.parse_error_with_context(format!( "Expected newline or next list item after list item {}", - i + 1 - ))); - } - } else if matches!(self.current_token, Token::Newline) { - // After the last item, check for extra items - self.advance()?; - self.skip_newlines()?; - - let list_indent = self.options.indent.get_spaces() * (depth + 1); - let actual_indent = self.scanner.get_last_line_indent(); - // If we see another dash at the same indent, there are too many items - if actual_indent == list_indent && matches!(self.current_token, Token::Dash) - { - return Err(self.parse_error_with_context(format!( - "Array length mismatch: expected {length} items, but more items \ - found", + items.len() ))); } } } } _ => { - for i in 0..length { - if i > 0 { - if matches!(self.current_token, Token::Delimiter(_)) { - self.advance()?; + if self.options.strict { + for i in 0..length { + if i > 0 { + if matches!(self.current_token, Token::Delimiter(_)) { + self.advance()?; + } else { + return Err(self + .parse_error_with_context(format!( + "Expected delimiter, found {:?}", + self.current_token + )) + .with_suggestion(format!( + "Expected delimiter between items (item {} of {})", + i + 1, + length + ))); + } + } + + let value = if matches!(self.current_token, Token::Delimiter(_)) + || (matches!(self.current_token, Token::Eof | Token::Newline) + && i < length) + { + Value::String(String::new()) + } else if matches!(self.current_token, Token::LeftBracket) { + self.parse_array(depth + 1)? } else { - return Err(self - .parse_error_with_context(format!( + self.parse_primitive()? + }; + + items.push(value); + } + } else { + let mut i = 0; + loop { + if i == 0 && matches!(self.current_token, Token::Newline | Token::Eof) { + break; + } + + if i > 0 { + if matches!(self.current_token, Token::Delimiter(_)) { + self.advance()?; + } else { + return Err(self.parse_error_with_context(format!( "Expected delimiter, found {:?}", self.current_token - )) - .with_suggestion(format!( - "Expected delimiter between items (item {} of {})", - i + 1, - length ))); + } } - } - let value = if matches!(self.current_token, Token::Delimiter(_)) - || (matches!(self.current_token, Token::Eof | Token::Newline) && i < length) - { - Value::String(String::new()) - } else if matches!(self.current_token, Token::LeftBracket) { - self.parse_array(depth + 1)? - } else { - self.parse_primitive()? - }; + let value = if matches!(self.current_token, Token::Delimiter(_)) + || matches!(self.current_token, Token::Eof | Token::Newline) + { + Value::String(String::new()) + } else if matches!(self.current_token, Token::LeftBracket) { + self.parse_array(depth + 1)? + } else { + self.parse_primitive()? + }; - items.push(value); + items.push(value); + i += 1; + + if matches!(self.current_token, Token::Newline | Token::Eof) { + break; + } + } } } } - validation::validate_array_length(length, items.len())?; + if self.options.strict { + validation::validate_array_length(length, items.len())?; - if self.options.strict && matches!(self.current_token, Token::Delimiter(_)) { - return Err(self.parse_error_with_context(format!( - "Array length mismatch: expected {length} items, but more items found", - ))); + if matches!(self.current_token, Token::Delimiter(_)) { + return Err(self.parse_error_with_context(format!( + "Array length mismatch: expected {length} items, but more items found", + ))); + } } Ok(Value::Array(items)) @@ -1438,6 +1696,19 @@ impl<'a> Parser<'a> { Ok(()) } } + + fn normalize_indent(&self, indent_amount: usize) -> usize { + if self.options.strict { + return indent_amount; + } + + let indent_size = self.options.indent.get_spaces(); + if indent_size == 0 { + indent_amount + } else { + (indent_amount / indent_size) * indent_size + } + } } #[cfg(test)] diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index bf71f13..c89a366 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -1,4 +1,7 @@ -use crate::types::{Delimiter, ToonError, ToonResult}; +use crate::{ + constants::DEFAULT_INDENT, + types::{Delimiter, ToonError, ToonResult}, +}; /// Tokens produced by the scanner during lexical analysis. #[derive(Debug, Clone, PartialEq)] @@ -28,6 +31,8 @@ pub struct Scanner { active_delimiter: Option, last_line_indent: usize, coerce_types: bool, + indent_width: usize, + allow_tab_indent: bool, } impl Scanner { @@ -41,6 +46,8 @@ impl Scanner { active_delimiter: None, last_line_indent: 0, coerce_types: true, + indent_width: DEFAULT_INDENT, + allow_tab_indent: false, } } @@ -53,6 +60,11 @@ impl Scanner { self.coerce_types = coerce_types; } + pub fn configure_indentation(&mut self, strict: bool, indent_width: usize) { + self.allow_tab_indent = !strict; + self.indent_width = indent_width.max(1); + } + /// Get the current position (line, column). pub fn current_position(&self) -> (usize, usize) { (self.line, self.column) @@ -71,17 +83,7 @@ impl Scanner { } pub fn count_leading_spaces(&self) -> usize { - let mut idx = self.position; - let mut count = 0; - while let Some(&ch) = self.input.get(idx) { - if ch == ' ' { - count += 1; - idx += 1; - } else { - break; - } - } - count + self.count_indent_from(self.position) } pub fn count_spaces_after_newline(&self) -> usize { @@ -90,16 +92,7 @@ impl Scanner { return 0; } idx += 1; - let mut count = 0; - while let Some(&ch) = self.input.get(idx) { - if ch == ' ' { - count += 1; - idx += 1; - } else { - break; - } - } - count + self.count_indent_from(idx) } pub fn peek_ahead(&self, offset: usize) -> Option { @@ -131,26 +124,47 @@ impl Scanner { } } + fn count_indent_from(&self, mut idx: usize) -> usize { + let mut count = 0; + while let Some(&ch) = self.input.get(idx) { + match ch { + ' ' => { + count += 1; + idx += 1; + } + '\t' if self.allow_tab_indent => { + count += self.indent_width; + idx += 1; + } + _ => break, + } + } + count + } + /// Scan the next token from the input. pub fn scan_token(&mut self) -> ToonResult { if self.column == 1 { let mut count = 0; - let mut idx = self.position; - - while let Some(&ch) = self.input.get(idx) { - if ch == ' ' { - count += 1; - idx += 1; - } else { - if ch == '\t' { - let (line, col) = self.current_position(); - return Err(ToonError::parse_error( - line, - col + count, - "Tabs are not allowed in indentation", - )); + while let Some(ch) = self.peek() { + match ch { + ' ' => { + count += 1; + self.advance(); } - break; + '\t' => { + if !self.allow_tab_indent { + let (line, col) = self.current_position(); + return Err(ToonError::parse_error( + line, + col + count, + "Tabs are not allowed in indentation", + )); + } + count += self.indent_width; + self.advance(); + } + _ => break, } } self.last_line_indent = count; diff --git a/src/decode/validation.rs b/src/decode/validation.rs index 9ff36b9..29aab52 100644 --- a/src/decode/validation.rs +++ b/src/decode/validation.rs @@ -1,14 +1,24 @@ use crate::types::{ToonError, ToonResult}; +use std::collections::HashSet; /// Validate that array length matches expected value. pub fn validate_array_length(expected: usize, actual: usize) -> ToonResult<()> { - // Array length mismatches should always error, regardless of strict mode if expected != actual { return Err(ToonError::length_mismatch(expected, actual)); } Ok(()) } +/// Validate that array length is non-negative. +pub fn validate_array_length_non_negative(length: i64) -> ToonResult<()> { + if length < 0 { + return Err(ToonError::InvalidInput( + "Array length must be non-negative".to_string(), + )); + } + Ok(()) +} + /// Validate field list for tabular arrays (no duplicates, non-empty names). pub fn validate_field_list(fields: &[String]) -> ToonResult<()> { if fields.is_empty() { @@ -17,24 +27,18 @@ pub fn validate_field_list(fields: &[String]) -> ToonResult<()> { )); } - // Check for duplicate field names - for i in 0..fields.len() { - for j in (i + 1)..fields.len() { - if fields[i] == fields[j] { - return Err(ToonError::InvalidInput(format!( - "Duplicate field name: '{}'", - fields[i] - ))); - } - } - } - + let mut seen = HashSet::with_capacity(fields.len()); for field in fields { if field.is_empty() { return Err(ToonError::InvalidInput( "Field name cannot be empty".to_string(), )); } + if !seen.insert(field.as_str()) { + return Err(ToonError::InvalidInput(format!( + "Duplicate field name: '{field}'" + ))); + } } Ok(()) @@ -80,6 +84,13 @@ mod tests { assert!(validate_array_length(5, 5).is_ok()); } + #[test] + fn test_validate_array_length_non_negative() { + assert!(validate_array_length_non_negative(0).is_ok()); + assert!(validate_array_length_non_negative(5).is_ok()); + assert!(validate_array_length_non_negative(-1).is_err()); + } + #[test] fn test_validate_field_list() { assert!(validate_field_list(&["id".to_string(), "name".to_string()]).is_ok()); diff --git a/src/types/value.rs b/src/types/value.rs index 5a86e21..3e55d10 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -443,9 +443,8 @@ impl IndexMut<&str> for JsonValue { match self { JsonValue::Object(obj) => { let len = obj.len(); - obj.get_mut(key).unwrap_or_else(|| { - panic!("key '{key}' not found in object with {len} entries") - }) + obj.get_mut(key) + .unwrap_or_else(|| panic!("key '{key}' not found in object with {len} entries")) } _ => panic!( "cannot index into non-object value of type {}", diff --git a/src/utils/mod.rs b/src/utils/mod.rs index f8ad2f1..1b300cb 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,7 +6,6 @@ pub mod validation; use indexmap::IndexMap; pub use literal::{is_keyword, is_literal_like, is_numeric_like, is_structural_char}; pub use number::format_canonical_number; -pub(crate) use string::is_valid_unquoted_key_decode; pub use string::{ escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, }; diff --git a/src/utils/string.rs b/src/utils/string.rs index fc0b2a9..8924442 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -114,10 +114,6 @@ pub fn is_valid_unquoted_key(key: &str) -> bool { is_valid_unquoted_key_internal(key, false) } -pub(crate) fn is_valid_unquoted_key_decode(key: &str) -> bool { - is_valid_unquoted_key_internal(key, true) -} - /// Determine if a string needs quoting based on content and delimiter. pub fn needs_quoting(s: &str, delimiter: char) -> bool { if s.is_empty() { @@ -303,6 +299,5 @@ mod tests { assert!(is_valid_unquoted_key("key.")); assert!(!is_valid_unquoted_key("key[value]")); assert!(!is_valid_unquoted_key("key{value}")); - assert!(is_valid_unquoted_key_decode("key-value")); } } diff --git a/src/utils/validation.rs b/src/utils/validation.rs index 72cdf84..f85aee9 100644 --- a/src/utils/validation.rs +++ b/src/utils/validation.rs @@ -5,9 +5,9 @@ use crate::types::{ToonError, ToonResult}; /// Validate that nesting depth doesn't exceed the maximum. pub fn validate_depth(depth: usize, max_depth: usize) -> ToonResult<()> { if depth > max_depth { - return Err(ToonError::InvalidStructure( - "Maximum nesting depth of {max_depth} exceeded".to_string(), - )); + return Err(ToonError::InvalidStructure(format!( + "Maximum nesting depth of {max_depth} exceeded" + ))); } Ok(()) } diff --git a/tests/strict_mode.rs b/tests/strict_mode.rs new file mode 100644 index 0000000..f16850c --- /dev/null +++ b/tests/strict_mode.rs @@ -0,0 +1,67 @@ +use serde_json::json; +use toon_format::{decode, DecodeOptions}; + +#[test] +fn test_negative_array_length_rejected() { + let input = "items[-1]:"; + let opts = DecodeOptions::new().with_strict(true); + let result = decode::(input, &opts); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("non-negative")); +} + +#[test] +fn test_float_array_length_rejected() { + let input = "items[3.5]:"; + let opts = DecodeOptions::new().with_strict(true); + let result = decode::(input, &opts); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("integer")); +} + +#[test] +fn test_mixed_delimiters_rejected_in_strict_mode() { + let input = "items[3]: a,b|c"; + let opts = DecodeOptions::new().with_strict(true); + let result = decode::(input, &opts); + + assert!(result.is_err()); +} + +#[test] +fn test_length_mismatch_allowed_in_non_strict_inline() { + let input = "items[1]: a,b"; + let opts = DecodeOptions::new().with_strict(false); + let result = decode::(input, &opts).unwrap(); + + assert_eq!(result["items"], json!(["a", "b"])); +} + +#[test] +fn test_length_mismatch_allowed_in_non_strict_list() { + let input = "items[1]:\n - 1\n - 2"; + let opts = DecodeOptions::new().with_strict(false); + let result = decode::(input, &opts).unwrap(); + + assert_eq!(result["items"], json!([1, 2])); +} + +#[test] +fn test_tab_indentation_allowed_in_non_strict_mode() { + let input = "items[1]:\n\t- 1"; + let opts = DecodeOptions::new().with_strict(false); + let result = decode::(input, &opts).unwrap(); + + assert_eq!(result["items"], json!([1])); +} + +#[test] +fn test_unquoted_key_rejected_in_strict_mode() { + let input = "bad-key: 1"; + let opts = DecodeOptions::new().with_strict(true); + let result = decode::(input, &opts); + + assert!(result.is_err()); +} From 789c11d51927460343d274ef37258ac24cb4af21 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 15:51:06 +0000 Subject: [PATCH 13/24] chore: split cli/tui feature flags --- Cargo.toml | 27 ++-- README.md | 16 ++ src/cli/main.rs | 80 ++++++---- src/lib.rs | 2 +- src/tui/app.rs | 231 ++++++++++++++++++---------- src/tui/components/history_panel.rs | 36 +++-- src/tui/state/file_state.rs | 43 +++++- src/tui/state/mod.rs | 2 +- 8 files changed, 291 insertions(+), 146 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1fa1015..47ffe49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,20 +25,13 @@ path = "src/cli/main.rs" required-features = ["cli"] [features] -default = ["cli"] -cli = [ - "dep:clap", - "dep:anyhow", - "dep:tiktoken-rs", - "dep:comfy-table", - "dep:ratatui", - "dep:crossterm", - "dep:tui-textarea", - "dep:arboard", - "dep:syntect", - "dep:unicode-width", - "dep:chrono", -] +default = ["cli", "cli-stats", "tui", "tui-clipboard", "tui-time", "parallel"] +cli = ["dep:clap", "dep:anyhow"] +cli-stats = ["cli", "dep:tiktoken-rs", "dep:comfy-table"] +tui = ["dep:anyhow", "dep:ratatui", "dep:crossterm", "dep:tui-textarea"] +tui-clipboard = ["tui", "dep:arboard"] +tui-time = ["tui", "dep:chrono"] +parallel = [] [dependencies] serde = { version = "1.0.228", features = ["derive"] } @@ -46,19 +39,17 @@ indexmap = "2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } thiserror = "2.0.17" -# CLI dependencies (gated behind "cli" feature) +# CLI dependencies (gated behind "cli"/"cli-stats" features) clap = { version = "4.5.11", features = ["derive"], optional = true } anyhow = { version = "1.0.86", optional = true } tiktoken-rs = { version = "0.9.1", optional = true } comfy-table = { version = "7.1", optional = true } -# TUI dependencies (gated behind "cli" feature) +# TUI dependencies (gated behind "tui" feature) ratatui = { version = "0.29", optional = true } crossterm = { version = "0.28", optional = true } tui-textarea = { version = "0.7", optional = true } arboard = { version = "3.4", optional = true } -syntect = { version = "5.2", optional = true } -unicode-width = { version = "0.2", optional = true } chrono = { version = "0.4", optional = true } [dev-dependencies] diff --git a/README.md b/README.md index b42d6d7..2e7d1c6 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,22 @@ cargo add toon-format cargo install toon-format ``` +### Feature Flags + +By default, all CLI/TUI features are enabled. You can opt in to only what you need: + +```toml +toon-format = { version = "0.4", default-features = false } +``` + +```bash +cargo install toon-format --no-default-features --features cli +cargo install toon-format --no-default-features --features cli,cli-stats +cargo install toon-format --no-default-features --features cli,tui,tui-clipboard,tui-time +``` + +Feature summary: `cli`, `cli-stats`, `tui`, `tui-clipboard`, `tui-time`, `parallel`. + --- ## Library Usage diff --git a/src/cli/main.rs b/src/cli/main.rs index d734f53..5115956 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -6,8 +6,10 @@ use std::{ use anyhow::{bail, Context, Result}; use clap::Parser; +#[cfg(feature = "cli-stats")] use comfy_table::Table; use serde::Serialize; +#[cfg(feature = "cli-stats")] use tiktoken_rs::cl100k_base; use toon_format::{ decode, encode, @@ -168,34 +170,7 @@ fn run_encode(cli: &Cli, input: &str) -> Result<()> { write_output(cli.output.clone(), &toon_str)?; if cli.stats { - let json_bytes = input.len(); - let toon_bytes = toon_str.len(); - let size_savings = 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64)); - - let bpe = cl100k_base().context("Failed to load tokenizer")?; - let json_tokens = bpe.encode_with_special_tokens(input).len(); - let toon_tokens = bpe.encode_with_special_tokens(&toon_str).len(); - let token_savings = 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64)); - - eprintln!("\nStats:"); - let mut table = Table::new(); - table.set_header(vec!["Metric", "JSON", "TOON", "Savings"]); - - table.add_row(vec![ - "Tokens", - &json_tokens.to_string(), - &toon_tokens.to_string(), - &format!("{token_savings:.2}%"), - ]); - - table.add_row(vec![ - "Size (bytes)", - &json_bytes.to_string(), - &toon_bytes.to_string(), - &format!("{size_savings:.2}%"), - ]); - - eprintln!("\n{table}\n"); + render_stats(input, &toon_str)?; } Ok(()) @@ -290,6 +265,11 @@ fn determine_operation(cli: &Cli) -> Result<(Operation, bool)> { } fn validate_flags(cli: &Cli, operation: &Operation) -> Result<()> { + #[cfg(not(feature = "cli-stats"))] + if cli.stats { + bail!("--stats requires the 'cli-stats' feature"); + } + match operation { Operation::Encode => { if cli.no_strict { @@ -359,6 +339,50 @@ fn main() -> Result<()> { Ok(()) } +#[cfg(feature = "cli-stats")] +fn render_stats(input: &str, toon_str: &str) -> Result<()> { + let json_bytes = input.len(); + let toon_bytes = toon_str.len(); + let size_savings = 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64)); + + let bpe = cl100k_base().context("Failed to load tokenizer")?; + let json_tokens = bpe.encode_with_special_tokens(input).len(); + let toon_tokens = bpe.encode_with_special_tokens(toon_str).len(); + let token_savings = 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64)); + + eprintln!("\nStats:"); + let mut table = Table::new(); + table.set_header(vec!["Metric", "JSON", "TOON", "Savings"]); + + table.add_row(vec![ + "Tokens", + &json_tokens.to_string(), + &toon_tokens.to_string(), + &format!("{token_savings:.2}%"), + ]); + + table.add_row(vec![ + "Size (bytes)", + &json_bytes.to_string(), + &toon_bytes.to_string(), + &format!("{size_savings:.2}%"), + ]); + + eprintln!("\n{table}\n"); + Ok(()) +} + +#[cfg(not(feature = "cli-stats"))] +fn render_stats(_input: &str, _toon_str: &str) -> Result<()> { + bail!("--stats requires the 'cli-stats' feature"); +} + +#[cfg(not(feature = "tui"))] +fn run_interactive() -> Result<()> { + bail!("Interactive mode requires the 'tui' feature"); +} + +#[cfg(feature = "tui")] fn run_interactive() -> Result<()> { toon_format::tui::run().context("Failed to run interactive TUI")?; Ok(()) diff --git a/src/lib.rs b/src/lib.rs index ce59d22..767488c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ pub mod constants; pub mod decode; pub mod encode; -#[cfg(feature = "cli")] +#[cfg(feature = "tui")] pub mod tui; pub mod types; pub mod utils; diff --git a/src/tui/app.rs b/src/tui/app.rs index 7b23c31..9c74637 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -1,8 +1,8 @@ use std::{fs, path::PathBuf, time::Duration}; use anyhow::{Context, Result}; -use chrono::Local; use crossterm::event::{KeyCode, KeyEvent}; +#[cfg(feature = "cli-stats")] use tiktoken_rs::cl100k_base; use crate::{ @@ -12,11 +12,14 @@ use crate::{ events::{Event, EventHandler}, keybindings::{Action, KeyBindings}, repl_command::ReplCommand, - state::{app_state::ConversionStats, AppState, ConversionHistory}, + state::{now_timestamp, AppState, ConversionHistory}, ui, }, }; +#[cfg(feature = "cli-stats")] +use crate::tui::state::ConversionStats; + /// Main TUI application managing state, events, and rendering. pub struct TuiApp<'a> { pub app_state: AppState<'a>, @@ -291,34 +294,48 @@ impl<'a> TuiApp<'a> { self.app_state.editor.set_output(toon_str.clone()); self.app_state.clear_error(); - if let Ok(bpe) = cl100k_base() { - let json_tokens = bpe.encode_with_special_tokens(input).len(); - let toon_tokens = bpe.encode_with_special_tokens(&toon_str).len(); - let json_bytes = input.len(); - let toon_bytes = toon_str.len(); - - let token_savings = - 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64)); - let byte_savings = 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64)); - - self.app_state.stats = Some(ConversionStats { - json_tokens, - toon_tokens, - json_bytes, - toon_bytes, - token_savings, - byte_savings, - }); - - self.app_state.file_state.add_to_history(ConversionHistory { - timestamp: Local::now(), - mode: "Encode".to_string(), - input_file: self.app_state.file_state.current_file.clone(), - output_file: None, - token_savings, - byte_savings, - }); + let json_bytes = input.len(); + let toon_bytes = toon_str.len(); + let byte_savings = if json_bytes == 0 { + 0.0 + } else { + 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64)) + }; + + let mut history_entry = ConversionHistory { + timestamp: now_timestamp(), + mode: "Encode".to_string(), + input_file: self.app_state.file_state.current_file.clone(), + output_file: None, + token_savings: None, + byte_savings: Some(byte_savings), + }; + + self.app_state.stats = None; + + #[cfg(feature = "cli-stats")] + { + if let Ok(bpe) = cl100k_base() { + let json_tokens = bpe.encode_with_special_tokens(input).len(); + let toon_tokens = bpe.encode_with_special_tokens(&toon_str).len(); + + let token_savings = + 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64)); + + self.app_state.stats = Some(ConversionStats { + json_tokens, + toon_tokens, + json_bytes, + toon_bytes, + token_savings, + byte_savings, + }); + + history_entry.token_savings = Some(token_savings); + } } + + self.app_state.file_state.add_to_history(history_entry); } Err(e) => { self.app_state.set_error(format!("Encode error: {e}")); @@ -339,34 +356,48 @@ impl<'a> TuiApp<'a> { self.app_state.editor.set_output(json_str.clone()); self.app_state.clear_error(); - if let Ok(bpe) = cl100k_base() { - let toon_tokens = bpe.encode_with_special_tokens(input).len(); - let json_tokens = bpe.encode_with_special_tokens(&json_str).len(); - let toon_bytes = input.len(); - let json_bytes = json_str.len(); - - let token_savings = - 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64)); - let byte_savings = 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64)); - - self.app_state.stats = Some(ConversionStats { - json_tokens, - toon_tokens, - json_bytes, - toon_bytes, - token_savings, - byte_savings, - }); - - self.app_state.file_state.add_to_history(ConversionHistory { - timestamp: Local::now(), - mode: "Decode".to_string(), - input_file: self.app_state.file_state.current_file.clone(), - output_file: None, - token_savings, - byte_savings, - }); + let toon_bytes = input.len(); + let json_bytes = json_str.len(); + let byte_savings = if toon_bytes == 0 { + 0.0 + } else { + 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64)) + }; + + let mut history_entry = ConversionHistory { + timestamp: now_timestamp(), + mode: "Decode".to_string(), + input_file: self.app_state.file_state.current_file.clone(), + output_file: None, + token_savings: None, + byte_savings: Some(byte_savings), + }; + + self.app_state.stats = None; + + #[cfg(feature = "cli-stats")] + { + if let Ok(bpe) = cl100k_base() { + let toon_tokens = bpe.encode_with_special_tokens(input).len(); + let json_tokens = bpe.encode_with_special_tokens(&json_str).len(); + + let token_savings = + 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64)); + + self.app_state.stats = Some(ConversionStats { + json_tokens, + toon_tokens, + json_bytes, + toon_bytes, + token_savings, + byte_savings, + }); + + history_entry.token_savings = Some(token_savings); + } } + + self.app_state.file_state.add_to_history(history_entry); } Err(e) => { self.app_state @@ -428,40 +459,60 @@ impl<'a> TuiApp<'a> { return Ok(()); } - #[cfg(not(target_os = "unknown"))] + #[cfg(feature = "tui-clipboard")] { - use arboard::Clipboard; - let mut clipboard = Clipboard::new()?; - clipboard.set_text(output)?; - self.app_state.set_status("Copied to clipboard".to_string()); + #[cfg(not(target_os = "unknown"))] + { + use arboard::Clipboard; + let mut clipboard = Clipboard::new()?; + clipboard.set_text(output)?; + self.app_state.set_status("Copied to clipboard".to_string()); + } + + #[cfg(target_os = "unknown")] + { + self.app_state + .set_error("Clipboard not supported on this platform".to_string()); + } } - #[cfg(target_os = "unknown")] + #[cfg(not(feature = "tui-clipboard"))] { - self.app_state - .set_error("Clipboard not supported on this platform".to_string()); + self.app_state.set_error( + "Clipboard support disabled (enable the 'tui-clipboard' feature)".to_string(), + ); } Ok(()) } fn paste_from_clipboard(&mut self) -> Result<()> { - #[cfg(not(target_os = "unknown"))] + #[cfg(feature = "tui-clipboard")] { - use arboard::Clipboard; - let mut clipboard = Clipboard::new()?; - let text = clipboard.get_text()?; - self.app_state.editor.set_input(text); - self.app_state.file_state.mark_modified(); - self.perform_conversion(); - self.app_state - .set_status("Pasted from clipboard".to_string()); + #[cfg(not(target_os = "unknown"))] + { + use arboard::Clipboard; + let mut clipboard = Clipboard::new()?; + let text = clipboard.get_text()?; + self.app_state.editor.set_input(text); + self.app_state.file_state.mark_modified(); + self.perform_conversion(); + self.app_state + .set_status("Pasted from clipboard".to_string()); + } + + #[cfg(target_os = "unknown")] + { + self.app_state + .set_error("Clipboard not supported on this platform".to_string()); + } } - #[cfg(target_os = "unknown")] + #[cfg(not(feature = "tui-clipboard"))] { - self.app_state - .set_error("Clipboard not supported on this platform".to_string()); + self.app_state.set_error( + "Clipboard support disabled (enable the 'tui-clipboard' feature)".to_string(), + ); } Ok(()) @@ -549,19 +600,29 @@ impl<'a> TuiApp<'a> { return Ok(()); } - #[cfg(not(target_os = "unknown"))] + #[cfg(feature = "tui-clipboard")] { - use arboard::Clipboard; - let mut clipboard = Clipboard::new()?; - clipboard.set_text(text)?; - self.app_state - .set_status("Copied selection to clipboard".to_string()); + #[cfg(not(target_os = "unknown"))] + { + use arboard::Clipboard; + let mut clipboard = Clipboard::new()?; + clipboard.set_text(text)?; + self.app_state + .set_status("Copied selection to clipboard".to_string()); + } + + #[cfg(target_os = "unknown")] + { + self.app_state + .set_error("Clipboard not supported on this platform".to_string()); + } } - #[cfg(target_os = "unknown")] + #[cfg(not(feature = "tui-clipboard"))] { - self.app_state - .set_error("Clipboard not supported on this platform".to_string()); + self.app_state.set_error( + "Clipboard support disabled (enable the 'tui-clipboard' feature)".to_string(), + ); } Ok(()) diff --git a/src/tui/components/history_panel.rs b/src/tui/components/history_panel.rs index 764a09d..273f310 100644 --- a/src/tui/components/history_panel.rs +++ b/src/tui/components/history_panel.rs @@ -7,7 +7,10 @@ use ratatui::{ Frame, }; -use crate::tui::{state::AppState, theme::Theme}; +use crate::tui::{ + state::{format_timestamp, AppState}, + theme::Theme, +}; pub struct HistoryPanel; @@ -48,7 +51,7 @@ impl HistoryPanel { .iter() .rev() .map(|entry| { - let time_str = entry.timestamp.format("%H:%M:%S").to_string(); + let time_str = format_timestamp(&entry.timestamp); let file_str = entry .input_file .as_ref() @@ -56,18 +59,33 @@ impl HistoryPanel { .and_then(|n| n.to_str()) .unwrap_or("stdin"); - ListItem::new(Line::from(vec![ - Span::styled(format!(" {time_str} "), theme.line_number_style()), - Span::styled(format!("[{}] ", entry.mode), theme.info_style()), - Span::styled(file_str, theme.normal_style()), - Span::styled( - format!(" → {:.1}% saved", entry.token_savings), - if entry.token_savings > 0.0 { + let (savings_text, savings_style) = match entry.token_savings { + Some(token_savings) => ( + format!(" → {:.1}% saved", token_savings), + if token_savings > 0.0 { theme.success_style() } else { theme.warning_style() }, ), + None => match entry.byte_savings { + Some(byte_savings) => ( + format!(" → {:.1}% bytes", byte_savings), + if byte_savings > 0.0 { + theme.success_style() + } else { + theme.warning_style() + }, + ), + None => (" → n/a".to_string(), theme.line_number_style()), + }, + }; + + ListItem::new(Line::from(vec![ + Span::styled(format!(" {time_str} "), theme.line_number_style()), + Span::styled(format!("[{}] ", entry.mode), theme.info_style()), + Span::styled(file_str, theme.normal_style()), + Span::styled(savings_text, savings_style), ])) }) .collect(); diff --git a/src/tui/state/file_state.rs b/src/tui/state/file_state.rs index 6467381..bdff523 100644 --- a/src/tui/state/file_state.rs +++ b/src/tui/state/file_state.rs @@ -2,15 +2,50 @@ use std::path::PathBuf; +#[cfg(feature = "tui-time")] use chrono::{DateTime, Local}; +#[cfg(feature = "tui-time")] +pub type Timestamp = DateTime; + +#[cfg(not(feature = "tui-time"))] +pub type Timestamp = (); + +pub fn now_timestamp() -> Option { + #[cfg(feature = "tui-time")] + { + Some(Local::now()) + } + + #[cfg(not(feature = "tui-time"))] + { + None + } +} + +pub fn format_timestamp(timestamp: &Option) -> String { + #[cfg(feature = "tui-time")] + { + timestamp + .as_ref() + .map(|ts| ts.format("%H:%M:%S").to_string()) + .unwrap_or_else(|| "--:--:--".to_string()) + } + + #[cfg(not(feature = "tui-time"))] + { + let _ = timestamp; + "--:--:--".to_string() + } +} + /// A file or directory entry. #[derive(Debug, Clone)] pub struct FileEntry { pub path: PathBuf, pub is_dir: bool, pub size: u64, - pub modified: Option>, + pub modified: Option, } impl FileEntry { @@ -34,12 +69,12 @@ impl FileEntry { /// Record of a conversion operation. #[derive(Debug, Clone)] pub struct ConversionHistory { - pub timestamp: DateTime, + pub timestamp: Option, pub mode: String, pub input_file: Option, pub output_file: Option, - pub token_savings: f64, - pub byte_savings: f64, + pub token_savings: Option, + pub byte_savings: Option, } /// File browser and conversion history state. diff --git a/src/tui/state/mod.rs b/src/tui/state/mod.rs index def4e5f..a3a5d52 100644 --- a/src/tui/state/mod.rs +++ b/src/tui/state/mod.rs @@ -7,5 +7,5 @@ pub mod repl_state; pub use app_state::{AppState, ConversionStats, Mode}; pub use editor_state::{EditorMode, EditorState}; -pub use file_state::{ConversionHistory, FileState}; +pub use file_state::{format_timestamp, now_timestamp, ConversionHistory, FileState}; pub use repl_state::{ReplLine, ReplLineKind, ReplState}; From 3ad150dd9359a3575bfeecbc10a18bbeccc5edeb Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 16:35:16 +0000 Subject: [PATCH 14/24] perf: throughput pass 1 --- Cargo.toml | 8 +- benches/encode_decode.rs | 92 ++++++++++++++ src/decode/expansion.rs | 109 ++++++---------- src/decode/parser.rs | 57 ++------- src/decode/scanner.rs | 67 +++++----- src/encode/folding.rs | 57 +++++---- src/encode/mod.rs | 47 ++++--- src/tui/state/app_state.rs | 69 ++++------ src/types/errors.rs | 250 +++++++++++++++++++++++++++++-------- src/utils/mod.rs | 26 ++++ tests/errors.rs | 6 +- 11 files changed, 502 insertions(+), 286 deletions(-) create mode 100644 benches/encode_decode.rs diff --git a/Cargo.toml b/Cargo.toml index 47ffe49..5d3918f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,13 +31,14 @@ cli-stats = ["cli", "dep:tiktoken-rs", "dep:comfy-table"] tui = ["dep:anyhow", "dep:ratatui", "dep:crossterm", "dep:tui-textarea"] tui-clipboard = ["tui", "dep:arboard"] tui-time = ["tui", "dep:chrono"] -parallel = [] +parallel = ["dep:rayon"] [dependencies] serde = { version = "1.0.228", features = ["derive"] } indexmap = "2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } thiserror = "2.0.17" +rayon = { version = "1.10", optional = true } # CLI dependencies (gated behind "cli"/"cli-stats" features) clap = { version = "4.5.11", features = ["derive"], optional = true } @@ -55,7 +56,12 @@ chrono = { version = "0.4", optional = true } [dev-dependencies] datatest-stable = "0.3.3" glob = "0.3" +criterion = "0.5" [[test]] name = "spec_fixtures" harness = false + +[[bench]] +name = "encode_decode" +harness = false diff --git a/benches/encode_decode.rs b/benches/encode_decode.rs new file mode 100644 index 0000000..ee97f99 --- /dev/null +++ b/benches/encode_decode.rs @@ -0,0 +1,92 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; + +fn make_tabular(rows: usize) -> Value { + let mut items = Vec::with_capacity(rows); + for i in 0..rows { + items.push(json!({ + "id": i, + "name": format!("User_{i}"), + "score": i * 2, + "active": i % 2 == 0, + "tag": format!("tag{i}"), + })); + } + Value::Array(items) +} + +fn make_deep_object(depth: usize) -> Value { + let mut value = json!({ + "leaf": "value", + "count": 1, + }); + + for i in 0..depth { + value = json!({ + format!("level_{i}"): value, + }); + } + + value +} + +fn make_long_unquoted(words: usize) -> String { + let mut parts = Vec::with_capacity(words); + for i in 0..words { + parts.push(format!("word{i}")); + } + parts.join(" ") +} + +fn bench_tabular(c: &mut Criterion) { + let mut group = c.benchmark_group("tabular"); + for rows in [128_usize, 1024] { + let value = make_tabular(rows); + let toon = encode_default(&value).expect("encode tabular"); + + group.bench_with_input(BenchmarkId::new("encode", rows), &value, |b, val| { + b.iter(|| encode_default(black_box(val)).expect("encode tabular")); + }); + + group.bench_with_input(BenchmarkId::new("decode", rows), &toon, |b, input| { + b.iter(|| decode_default::(black_box(input)).expect("decode tabular")); + }); + } + group.finish(); +} + +fn bench_deep_object(c: &mut Criterion) { + let mut group = c.benchmark_group("deep_object"); + for depth in [32_usize, 128] { + let value = make_deep_object(depth); + let toon = encode_default(&value).expect("encode deep object"); + + group.bench_with_input(BenchmarkId::new("encode", depth), &value, |b, val| { + b.iter(|| encode_default(black_box(val)).expect("encode deep object")); + }); + + group.bench_with_input(BenchmarkId::new("decode", depth), &toon, |b, input| { + b.iter(|| decode_default::(black_box(input)).expect("decode deep object")); + }); + } + group.finish(); +} + +fn bench_long_unquoted(c: &mut Criterion) { + let words = 512; + let long_value = make_long_unquoted(words); + let toon = format!("value: {long_value}"); + + c.bench_function("decode_long_unquoted", |b| { + b.iter(|| decode_default::(black_box(&toon)).expect("decode long unquoted")); + }); +} + +criterion_group!( + benches, + bench_tabular, + bench_deep_object, + bench_long_unquoted +); +criterion_main!(benches); diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index d665d45..4162d24 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -5,7 +5,7 @@ use crate::{ types::{is_identifier_segment, JsonValue as Value, PathExpansionMode, ToonError, ToonResult}, }; -pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option> { +pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option> { match mode { PathExpansionMode::Off => None, PathExpansionMode::Safe => { @@ -18,7 +18,7 @@ pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option = key.split('.').map(String::from).collect(); + let segments: Vec<&str> = key.split('.').collect(); if segments.len() < 2 { return None; @@ -36,7 +36,7 @@ pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option, - segments: &[String], + segments: &[&str], value: Value, strict: bool, ) -> ToonResult<()> { @@ -45,7 +45,7 @@ pub fn deep_merge_value( } if segments.len() == 1 { - let key = &segments[0]; + let key = segments[0]; // Check for conflicts at leaf level if let Some(existing) = target.get(key) { @@ -56,11 +56,11 @@ pub fn deep_merge_value( } } - target.insert(key.clone(), value); + target.insert(key.to_string(), value); return Ok(()); } - let first_key = &segments[0]; + let first_key = segments[0]; let remaining_segments = &segments[1..]; // Get or create nested object, handling type conflicts @@ -74,7 +74,6 @@ pub fn deep_merge_value( {existing_value:?}", ))); } - // Replace non-object with empty object in non-strict mode *existing_value = Value::Object(IndexMap::new()); match existing_value { Value::Object(obj) => obj, @@ -83,7 +82,7 @@ pub fn deep_merge_value( } } } else { - target.insert(first_key.clone(), Value::Object(IndexMap::new())); + target.insert(first_key.to_string(), Value::Object(IndexMap::new())); match target.get_mut(first_key).expect("key was just inserted") { Value::Object(obj) => obj, _ => unreachable!(), @@ -99,31 +98,34 @@ pub fn expand_paths_in_object( mode: PathExpansionMode, strict: bool, ) -> ToonResult> { - let mut result = IndexMap::new(); + let mut result = IndexMap::with_capacity(obj.len()); for (key, mut value) in obj { // Expand nested structures (arrays/objects) first (depth-first) value = expand_paths_recursive(value, mode, strict)?; - // Strip marker from quoted keys - let clean_key = if key.starts_with(QUOTED_KEY_MARKER) { - key.strip_prefix(QUOTED_KEY_MARKER).unwrap().to_string() - } else { - key.clone() - }; - - if let Some(segments) = should_expand_key(&key, mode) { - deep_merge_value(&mut result, &segments, value, strict)?; - } else { - // Check for conflicts with expanded keys - if let Some(existing) = result.get(&clean_key) { - if strict { - return Err(ToonError::DeserializationError(format!( - "Key '{clean_key}' conflicts with existing value: {existing:?}", - ))); + match should_expand_key(&key, mode) { + Some(segments) => { + deep_merge_value(&mut result, &segments, value, strict)?; + } + None => { + // Strip marker from quoted keys + let clean_key = if key.starts_with(QUOTED_KEY_MARKER) { + key.strip_prefix(QUOTED_KEY_MARKER).unwrap().to_string() + } else { + key + }; + + // Check for conflicts with expanded keys + if let Some(existing) = result.get(clean_key.as_str()) { + if strict { + return Err(ToonError::DeserializationError(format!( + "Key '{clean_key}' conflicts with existing value: {existing:?}", + ))); + } } + result.insert(clean_key, value); } - result.insert(clean_key, value); } } @@ -141,11 +143,11 @@ pub fn expand_paths_recursive( Ok(Value::Object(expanded)) } Value::Array(arr) => { - let expanded: Result, _> = arr - .into_iter() - .map(|v| expand_paths_recursive(v, mode, strict)) - .collect(); - Ok(Value::Array(expanded?)) + let mut expanded = Vec::with_capacity(arr.len()); + for item in arr { + expanded.push(expand_paths_recursive(item, mode, strict)?); + } + Ok(Value::Array(expanded)) } _ => Ok(value), } @@ -167,11 +169,11 @@ mod tests { // Valid expansions assert_eq!( should_expand_key("a.b", PathExpansionMode::Safe), - Some(vec!["a".to_string(), "b".to_string()]) + Some(vec!["a", "b"]) ); assert_eq!( should_expand_key("a.b.c", PathExpansionMode::Safe), - Some(vec!["a".to_string(), "b".to_string(), "c".to_string()]) + Some(vec!["a", "b", "c"]) ); // No dots @@ -185,13 +187,7 @@ mod tests { #[test] fn test_deep_merge_simple() { let mut target = IndexMap::new(); - deep_merge_value( - &mut target, - &["a".to_string(), "b".to_string()], - Value::from(json!(1)), - true, - ) - .unwrap(); + deep_merge_value(&mut target, &["a", "b"], Value::from(json!(1)), true).unwrap(); let expected = json!({"a": {"b": 1}}); assert_eq!(Value::Object(target), Value::from(expected)); @@ -201,21 +197,9 @@ mod tests { fn test_deep_merge_multiple_paths() { let mut target = IndexMap::new(); - deep_merge_value( - &mut target, - &["a".to_string(), "b".to_string()], - Value::from(json!(1)), - true, - ) - .unwrap(); - - deep_merge_value( - &mut target, - &["a".to_string(), "c".to_string()], - Value::from(json!(2)), - true, - ) - .unwrap(); + deep_merge_value(&mut target, &["a", "b"], Value::from(json!(1)), true).unwrap(); + + deep_merge_value(&mut target, &["a", "c"], Value::from(json!(2)), true).unwrap(); let expected = json!({"a": {"b": 1, "c": 2}}); assert_eq!(Value::Object(target), Value::from(expected)); @@ -226,12 +210,7 @@ mod tests { let mut target = IndexMap::new(); target.insert("a".to_string(), Value::from(json!({"b": 1}))); - let result = deep_merge_value( - &mut target, - &["a".to_string(), "b".to_string()], - Value::from(json!(2)), - true, - ); + let result = deep_merge_value(&mut target, &["a", "b"], Value::from(json!(2)), true); assert!(result.is_err()); } @@ -241,13 +220,7 @@ mod tests { let mut target = IndexMap::new(); target.insert("a".to_string(), Value::from(json!({"b": 1}))); - deep_merge_value( - &mut target, - &["a".to_string(), "b".to_string()], - Value::from(json!(2)), - false, - ) - .unwrap(); + deep_merge_value(&mut target, &["a", "b"], Value::from(json!(2)), false).unwrap(); let expected = json!({"a": {"b": 2}}); assert_eq!(Value::Object(target), Value::from(expected)); diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 38df222..795e9b3 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -1,4 +1,5 @@ use serde_json::{Map, Number, Value}; +use std::sync::Arc; use crate::{ constants::{KEYWORDS, MAX_DEPTH, QUOTED_KEY_MARKER}, @@ -27,19 +28,20 @@ enum ArrayParseContext { /// Parser that builds JSON values from a sequence of tokens. #[allow(unused)] -pub struct Parser<'a> { +pub struct Parser { scanner: Scanner, current_token: Token, options: DecodeOptions, delimiter: Option, delimiter_stack: Vec>, - input: &'a str, + input: Arc, } -impl<'a> Parser<'a> { +impl Parser { /// Create a new parser with the given input and options. - pub fn new(input: &'a str, options: DecodeOptions) -> ToonResult { - let mut scanner = Scanner::new(input); + pub fn new(input: &str, options: DecodeOptions) -> ToonResult { + let input: Arc = Arc::from(input); + let mut scanner = Scanner::from_shared_input(input.clone()); let chosen_delim = options.delimiter; scanner.set_active_delimiter(chosen_delim); scanner.set_coerce_types(options.coerce_types); @@ -1628,7 +1630,8 @@ impl<'a> Parser<'a> { let (line, column) = self.scanner.current_position(); let message = message.into(); - let context = self.get_error_context(line, column); + let context = ErrorContext::from_shared_input(self.input.clone(), line, column, 2) + .unwrap_or_else(|| ErrorContext::new("")); ToonError::ParseError { line, @@ -1638,48 +1641,6 @@ impl<'a> Parser<'a> { } } - fn get_error_context(&self, line: usize, column: usize) -> ErrorContext { - let lines: Vec<&str> = self.input.lines().collect(); - - let source_line = if line > 0 && line <= lines.len() { - lines[line - 1].to_string() - } else { - String::new() - }; - - let preceding_lines: Vec = if line > 1 { - lines[line.saturating_sub(3)..line - 1] - .iter() - .map(|s| s.to_string()) - .collect() - } else { - Vec::new() - }; - - let following_lines: Vec = if line < lines.len() { - lines[line..line.saturating_add(2).min(lines.len())] - .iter() - .map(|s| s.to_string()) - .collect() - } else { - Vec::new() - }; - - let indicator = if column > 0 { - Some(format!("{:width$}^", "", width = column - 1)) - } else { - None - }; - - ErrorContext { - source_line, - preceding_lines, - following_lines, - suggestion: None, - indicator, - } - } - fn validate_indentation(&self, indent_amount: usize) -> ToonResult<()> { if !self.options.strict { return Ok(()); diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index c89a366..61cede4 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::{ constants::DEFAULT_INDENT, types::{Delimiter, ToonError, ToonResult}, @@ -24,7 +26,7 @@ pub enum Token { /// Scanner that tokenizes TOON input into a sequence of tokens. pub struct Scanner { - input: Vec, + input: Arc, position: usize, line: usize, column: usize, @@ -38,8 +40,12 @@ pub struct Scanner { impl Scanner { /// Create a new scanner for the given input string. pub fn new(input: &str) -> Self { + Self::from_shared_input(Arc::from(input)) + } + + pub fn from_shared_input(input: Arc) -> Self { Self { - input: input.chars().collect(), + input, position: 0, line: 1, column: 1, @@ -79,7 +85,7 @@ impl Scanner { } pub fn peek(&self) -> Option { - self.input.get(self.position).copied() + self.input[self.position..].chars().next() } pub fn count_leading_spaces(&self) -> usize { @@ -88,7 +94,7 @@ impl Scanner { pub fn count_spaces_after_newline(&self) -> usize { let mut idx = self.position; - if self.input.get(idx) != Some(&'\n') { + if self.input.as_bytes().get(idx) != Some(&b'\n') { return 0; } idx += 1; @@ -96,19 +102,19 @@ impl Scanner { } pub fn peek_ahead(&self, offset: usize) -> Option { - self.input.get(self.position + offset).copied() + self.input[self.position..].chars().nth(offset) } pub fn advance(&mut self) -> Option { - if let Some(ch) = self.input.get(self.position) { - self.position += 1; - if *ch == '\n' { + if let Some(ch) = self.peek() { + self.position += ch.len_utf8(); + if ch == '\n' { self.line += 1; self.column = 1; } else { self.column += 1; } - Some(*ch) + Some(ch) } else { None } @@ -126,13 +132,14 @@ impl Scanner { fn count_indent_from(&self, mut idx: usize) -> usize { let mut count = 0; - while let Some(&ch) = self.input.get(idx) { - match ch { - ' ' => { + let bytes = self.input.as_bytes(); + while idx < bytes.len() { + match bytes[idx] { + b' ' => { count += 1; idx += 1; } - '\t' if self.allow_tab_indent => { + b'\t' if self.allow_tab_indent => { count += self.indent_width; idx += 1; } @@ -307,11 +314,10 @@ impl Scanner { } // Single-char delimiters kept as-is, others trimmed - let value = if value.len() == 1 && (value == "," || value == "|" || value == "\t") { - value - } else { - value.trim_end().to_string() - }; + if !(value.len() == 1 && (value == "," || value == "|" || value == "\t")) { + let trimmed_len = value.trim_end().len(); + value.truncate(trimmed_len); + } if !self.coerce_types { return Ok(Token::String(value, false)); @@ -441,11 +447,11 @@ impl Scanner { if trimmed.starts_with('"') { let mut value = String::new(); let mut escaped = false; - let chars: Vec = trimmed.chars().collect(); - let mut i = 1; - while i < chars.len() { - let ch = chars[i]; + let mut chars = trimmed.char_indices(); + chars.next(); + + for (idx, ch) in chars { if escaped { match ch { 'n' => value.push('\n'), @@ -462,10 +468,16 @@ impl Scanner { } } escaped = false; - } else if ch == '\\' { + continue; + } + + if ch == '\\' { escaped = true; - } else if ch == '"' { - if i != chars.len() - 1 { + continue; + } + + if ch == '"' { + if idx + ch.len_utf8() != trimmed.len() { return Err(ToonError::parse_error( self.line, self.column, @@ -473,10 +485,9 @@ impl Scanner { )); } return Ok(Token::String(value, true)); - } else { - value.push(ch); } - i += 1; + + value.push(ch); } return Err(ToonError::parse_error( diff --git a/src/encode/folding.rs b/src/encode/folding.rs index 0034970..3ead315 100644 --- a/src/encode/folding.rs +++ b/src/encode/folding.rs @@ -1,42 +1,45 @@ +use std::collections::HashSet; + use crate::types::{is_identifier_segment, JsonValue as Value, KeyFoldingMode}; /// Result of chain analysis for folding. -pub struct FoldableChain { +pub struct FoldableChain<'a> { /// The folded key path (e.g., "a.b.c") pub folded_key: String, /// The leaf value at the end of the chain - pub leaf_value: Value, + pub leaf_value: &'a Value, /// Number of segments that were folded pub depth_folded: usize, } /// Check if a value is a single-key object suitable for folding. -fn is_single_key_object(value: &Value) -> Option<(&String, &Value)> { +fn is_single_key_object(value: &Value) -> Option<(&str, &Value)> { if let Value::Object(obj) = value { if obj.len() == 1 { - return obj.iter().next(); + return obj.iter().next().map(|(key, val)| (key.as_str(), val)); } } None } /// Analyze if a key-value pair can be folded into dotted notation. -pub fn analyze_foldable_chain( - key: &str, - value: &Value, +pub fn analyze_foldable_chain<'a>( + key: &'a str, + value: &'a Value, flatten_depth: usize, - existing_keys: &[&String], -) -> Option { + existing_keys: &HashSet<&str>, +) -> Option> { if !is_identifier_segment(key) { return None; } - let mut segments = vec![key.to_string()]; + let mut segments = Vec::with_capacity(4); + segments.push(key); let mut current_value = value; // Follow single-key object chain until we hit a multi-key object or leaf while let Some((next_key, next_value)) = is_single_key_object(current_value) { - if segments.len() >= flatten_depth { + if flatten_depth != usize::MAX && segments.len() >= flatten_depth { break; } @@ -44,7 +47,7 @@ pub fn analyze_foldable_chain( break; } - segments.push(next_key.clone()); + segments.push(next_key); current_value = next_value; } @@ -53,16 +56,24 @@ pub fn analyze_foldable_chain( return None; } - let folded_key = segments.join("."); + let total_len = + segments.iter().map(|segment| segment.len()).sum::() + segments.len() - 1; + let mut folded_key = String::with_capacity(total_len); + for (idx, segment) in segments.iter().enumerate() { + if idx > 0 { + folded_key.push('.'); + } + folded_key.push_str(segment); + } // Don't fold if it would collide with an existing key - if existing_keys.contains(&&folded_key) { + if existing_keys.contains(folded_key.as_str()) { return None; } Some(FoldableChain { folded_key, - leaf_value: current_value.clone(), + leaf_value: current_value, depth_folded: segments.len(), }) } @@ -77,6 +88,7 @@ pub fn should_fold(mode: KeyFoldingMode, chain: &Option) -> bool #[cfg(test)] mod tests { use serde_json::json; + use std::collections::HashSet; use super::*; @@ -95,7 +107,7 @@ mod tests { #[test] fn test_analyze_simple_chain() { let val = Value::from(json!({"b": {"c": 1}})); - let existing: Vec<&String> = vec![]; + let existing: HashSet<&str> = HashSet::new(); let result = analyze_foldable_chain("a", &val, usize::MAX, &existing); assert!(result.is_some()); @@ -103,13 +115,13 @@ mod tests { let chain = result.unwrap(); assert_eq!(chain.folded_key, "a.b.c"); assert_eq!(chain.depth_folded, 3); - assert_eq!(chain.leaf_value, Value::from(json!(1))); + assert_eq!(chain.leaf_value, &Value::from(json!(1))); } #[test] fn test_analyze_with_flatten_depth() { let val = Value::from(json!({"b": {"c": {"d": 1}}})); - let existing: Vec<&String> = vec![]; + let existing: HashSet<&str> = HashSet::new(); let result = analyze_foldable_chain("a", &val, 2, &existing); assert!(result.is_some()); @@ -122,7 +134,7 @@ mod tests { #[test] fn test_analyze_stops_at_multi_key() { let val = Value::from(json!({"b": {"c": 1, "d": 2}})); - let existing: Vec<&String> = vec![]; + let existing: HashSet<&str> = HashSet::new(); let result = analyze_foldable_chain("a", &val, usize::MAX, &existing); assert!(result.is_some()); @@ -135,7 +147,7 @@ mod tests { #[test] fn test_analyze_rejects_non_identifier() { let val = Value::from(json!({"c": 1})); - let existing: Vec<&String> = vec![]; + let existing: HashSet<&str> = HashSet::new(); let result = analyze_foldable_chain("bad-key", &val, usize::MAX, &existing); assert!(result.is_none()); @@ -145,7 +157,8 @@ mod tests { fn test_analyze_detects_collision() { let val = Value::from(json!({"b": 1})); let existing_key = String::from("a.b"); - let existing: Vec<&String> = vec![&existing_key]; + let mut existing: HashSet<&str> = HashSet::new(); + existing.insert(existing_key.as_str()); let result = analyze_foldable_chain("a", &val, usize::MAX, &existing); assert!(result.is_none()); @@ -154,7 +167,7 @@ mod tests { #[test] fn test_analyze_too_short_chain() { let val = Value::from(json!(42)); - let existing: Vec<&String> = vec![]; + let existing: HashSet<&str> = HashSet::new(); let result = analyze_foldable_chain("a", &val, usize::MAX, &existing); assert!(result.is_none()); diff --git a/src/encode/mod.rs b/src/encode/mod.rs index 69b2d1b..d812748 100644 --- a/src/encode/mod.rs +++ b/src/encode/mod.rs @@ -3,6 +3,7 @@ pub mod folding; pub mod primitives; pub mod writer; use indexmap::IndexMap; +use std::collections::HashSet; use crate::{ constants::MAX_DEPTH, @@ -59,11 +60,11 @@ pub fn encode(value: &T, options: &EncodeOptions) -> ToonRe let json_value = serde_json::to_value(value).map_err(|e| ToonError::SerializationError(e.to_string()))?; let json_value: Value = json_value.into(); - encode_impl(&json_value, options) + encode_impl(json_value, options) } -fn encode_impl(value: &Value, options: &EncodeOptions) -> ToonResult { - let normalized: Value = normalize(value.clone()); +fn encode_impl(value: Value, options: &EncodeOptions) -> ToonResult { + let normalized: Value = normalize(value); let mut writer = writer::Writer::new(options.clone()); match &normalized { @@ -148,7 +149,7 @@ pub fn encode_object(value: V, options: &EncodeOptions) -> Too found: value_type_name(&json_value).to_string(), }); } - encode_impl(&json_value, options) + encode_impl(json_value, options) } /// Encode a JSON array to TOON format (errors if not an array). @@ -178,7 +179,7 @@ pub fn encode_array(value: V, options: &EncodeOptions) -> Toon found: value_type_name(&json_value).to_string(), }); } - encode_impl(&json_value, options) + encode_impl(json_value, options) } fn value_type_name(value: &Value) -> &'static str { @@ -209,26 +210,42 @@ fn write_object_impl( validate_depth(depth, MAX_DEPTH)?; let keys: Vec<&String> = obj.keys().collect(); + let mut key_set: HashSet<&str> = HashSet::with_capacity(keys.len()); + let mut prefix_conflicts: HashSet<&str> = HashSet::new(); + + for key in &keys { + key_set.insert(key.as_str()); + if key.contains('.') { + let mut start = 0; + while let Some(pos) = key[start..].find('.') { + let end = start + pos; + if end > 0 { + prefix_conflicts.insert(&key[..end]); + } + start = end + 1; + } + } + } for (i, key) in keys.iter().enumerate() { if i > 0 { writer.write_newline()?; } - let value = obj.get(*key).expect("key exists in field list"); + let key = *key; + let key_str = key.as_str(); + let value = obj.get(key).expect("key exists in field list"); // Check if this key-value pair can be folded (v1.5 feature) // Don't fold if any sibling key is a dotted path starting with this key // (e.g., don't fold inside "data" if "data.meta.items" exists as a sibling) - let has_conflicting_sibling = keys - .iter() - .any(|k| k.starts_with(&format!("{key}.")) || (k.contains('.') && k == key)); + let has_conflicting_sibling = key_str.contains('.') || prefix_conflicts.contains(key_str); let folded = if !disable_folding && writer.options.key_folding == KeyFoldingMode::Safe && !has_conflicting_sibling { - folding::analyze_foldable_chain(key, value, writer.options.flatten_depth, &keys) + folding::analyze_foldable_chain(key_str, value, writer.options.flatten_depth, &key_set) } else { None }; @@ -240,7 +257,7 @@ fn write_object_impl( } // Write the leaf value - match &chain.leaf_value { + match chain.leaf_value { Value::Array(arr) => { // For arrays, pass the folded key to write_array so it generates the header // correctly @@ -262,20 +279,20 @@ fn write_object_impl( writer.write_key(&chain.folded_key)?; writer.write_char(':')?; writer.write_char(' ')?; - write_primitive_value(writer, &chain.leaf_value, QuotingContext::ObjectValue)?; + write_primitive_value(writer, chain.leaf_value, QuotingContext::ObjectValue)?; } } } else { // Standard (non-folded) encoding match value { Value::Array(arr) => { - write_array(writer, Some(key), arr, depth)?; + write_array(writer, Some(key_str), arr, depth)?; } Value::Object(nested_obj) => { if depth > 0 { writer.write_indent(depth)?; } - writer.write_key(key)?; + writer.write_key(key_str)?; writer.write_char(':')?; if !nested_obj.is_empty() { writer.write_newline()?; @@ -289,7 +306,7 @@ fn write_object_impl( if depth > 0 { writer.write_indent(depth)?; } - writer.write_key(key)?; + writer.write_key(key_str)?; writer.write_char(':')?; writer.write_char(' ')?; write_primitive_value(writer, value, QuotingContext::ObjectValue)?; diff --git a/src/tui/state/app_state.rs b/src/tui/state/app_state.rs index d4925b8..1ef8408 100644 --- a/src/tui/state/app_state.rs +++ b/src/tui/state/app_state.rs @@ -177,96 +177,73 @@ impl<'a> AppState<'a> { } pub fn cycle_delimiter(&mut self) { - self.encode_options = - self.encode_options - .clone() - .with_delimiter(match self.encode_options.delimiter { - Delimiter::Comma => Delimiter::Tab, - Delimiter::Tab => Delimiter::Pipe, - Delimiter::Pipe => Delimiter::Comma, - }); + self.encode_options.delimiter = match self.encode_options.delimiter { + Delimiter::Comma => Delimiter::Tab, + Delimiter::Tab => Delimiter::Pipe, + Delimiter::Pipe => Delimiter::Comma, + }; } pub fn increase_indent(&mut self) { let Indent::Spaces(current) = self.encode_options.indent; if current < 8 { - self.encode_options = self - .encode_options - .clone() - .with_indent(Indent::Spaces(current + 1)); + self.encode_options.indent = Indent::Spaces(current + 1); } } pub fn decrease_indent(&mut self) { let Indent::Spaces(current) = self.encode_options.indent; if current > 1 { - self.encode_options = self - .encode_options - .clone() - .with_indent(Indent::Spaces(current - 1)); + self.encode_options.indent = Indent::Spaces(current - 1); } } pub fn toggle_fold_keys(&mut self) { - self.encode_options = - self.encode_options - .clone() - .with_key_folding(match self.encode_options.key_folding { - KeyFoldingMode::Off => KeyFoldingMode::Safe, - KeyFoldingMode::Safe => KeyFoldingMode::Off, - }); + self.encode_options.key_folding = match self.encode_options.key_folding { + KeyFoldingMode::Off => KeyFoldingMode::Safe, + KeyFoldingMode::Safe => KeyFoldingMode::Off, + }; } pub fn increase_flatten_depth(&mut self) { if self.encode_options.flatten_depth == usize::MAX { - self.encode_options = self.encode_options.clone().with_flatten_depth(2); + self.encode_options.flatten_depth = 2; } else if self.encode_options.flatten_depth < 10 { - self.encode_options = self - .encode_options - .clone() - .with_flatten_depth(self.encode_options.flatten_depth + 1); + self.encode_options.flatten_depth += 1; } } pub fn decrease_flatten_depth(&mut self) { if self.encode_options.flatten_depth == 2 { - self.encode_options = self.encode_options.clone().with_flatten_depth(usize::MAX); + self.encode_options.flatten_depth = usize::MAX; } else if self.encode_options.flatten_depth > 2 && self.encode_options.flatten_depth != usize::MAX { - self.encode_options = self - .encode_options - .clone() - .with_flatten_depth(self.encode_options.flatten_depth - 1); + self.encode_options.flatten_depth -= 1; } } pub fn toggle_flatten_depth(&mut self) { if self.encode_options.flatten_depth == usize::MAX { - self.encode_options = self.encode_options.clone().with_flatten_depth(2); + self.encode_options.flatten_depth = 2; } else { - self.encode_options = self.encode_options.clone().with_flatten_depth(usize::MAX); + self.encode_options.flatten_depth = usize::MAX; } } pub fn toggle_expand_paths(&mut self) { - self.decode_options = - self.decode_options - .clone() - .with_expand_paths(match self.decode_options.expand_paths { - PathExpansionMode::Off => PathExpansionMode::Safe, - PathExpansionMode::Safe => PathExpansionMode::Off, - }); + self.decode_options.expand_paths = match self.decode_options.expand_paths { + PathExpansionMode::Off => PathExpansionMode::Safe, + PathExpansionMode::Safe => PathExpansionMode::Off, + }; } pub fn toggle_strict(&mut self) { - let strict = !self.decode_options.strict; - self.decode_options = self.decode_options.clone().with_strict(strict); + self.decode_options.strict = !self.decode_options.strict; } pub fn toggle_coerce_types(&mut self) { - let coerce = !self.decode_options.coerce_types; - self.decode_options = self.decode_options.clone().with_coerce_types(coerce); + self.decode_options.coerce_types = !self.decode_options.coerce_types; } } diff --git a/src/types/errors.rs b/src/types/errors.rs index ad85f33..5c3e70c 100644 --- a/src/types/errors.rs +++ b/src/types/errors.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use thiserror::Error; /// Result type alias for TOON operations. @@ -50,31 +51,83 @@ pub enum ToonError { /// Contextual information for error reporting, including source location /// and suggestions. +#[derive(Debug, Clone, PartialEq, Eq)] +enum ErrorContextSource { + Inline { + source_line: String, + preceding_lines: Vec, + following_lines: Vec, + indicator: Option, + }, + Lazy { + input: Arc, + line: usize, + column: usize, + context_lines: usize, + }, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ErrorContext { - pub source_line: String, - pub preceding_lines: Vec, - pub following_lines: Vec, + source: ErrorContextSource, pub suggestion: Option, - pub indicator: Option, } impl std::fmt::Display for ErrorContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(f, "\nContext:")?; - for line in &self.preceding_lines { - writeln!(f, " {line}")?; - } + match &self.source { + ErrorContextSource::Inline { + source_line, + preceding_lines, + following_lines, + indicator, + } => { + for line in preceding_lines { + writeln!(f, " {line}")?; + } - writeln!(f, "> {}", self.source_line)?; + writeln!(f, "> {source_line}")?; - if let Some(indicator) = &self.indicator { - writeln!(f, " {indicator}")?; - } + if let Some(indicator) = indicator { + writeln!(f, " {indicator}")?; + } + + for line in following_lines { + writeln!(f, " {line}")?; + } + } + ErrorContextSource::Lazy { + input, + line, + column, + context_lines, + } => { + let lines: Vec<&str> = input.lines().collect(); + if *line == 0 || *line > lines.len() { + return Ok(()); + } - for line in &self.following_lines { - writeln!(f, " {line}")?; + let line_idx = line - 1; + let start_line = line_idx.saturating_sub(*context_lines); + let end_line = (line_idx + context_lines + 1).min(lines.len()); + + for line in &lines[start_line..line_idx] { + writeln!(f, " {line}")?; + } + + writeln!(f, "> {}", lines[line_idx])?; + + if *column > 0 { + let indicator = " ".repeat(column.saturating_sub(1)); + writeln!(f, " {indicator}^")?; + } + + for line in &lines[(line_idx + 1)..end_line] { + writeln!(f, " {line}")?; + } + } } if let Some(suggestion) = &self.suggestion { @@ -87,27 +140,71 @@ impl std::fmt::Display for ErrorContext { impl std::error::Error for ErrorContext {} +fn extract_lines_to_strings( + input: &str, + line: usize, + context_lines: usize, +) -> (String, Vec, Vec) { + let lines: Vec<&str> = input.lines().collect(); + + if line == 0 || line > lines.len() { + return (String::new(), Vec::new(), Vec::new()); + } + + let line_idx = line - 1; + let source_line = lines.get(line_idx).unwrap_or(&"").to_string(); + + let start_line = line_idx.saturating_sub(context_lines); + let end_line = (line_idx + context_lines + 1).min(lines.len()); + + let preceding_lines = lines[start_line..line_idx] + .iter() + .map(|s| s.to_string()) + .collect(); + + let following_lines = lines[(line_idx + 1)..end_line] + .iter() + .map(|s| s.to_string()) + .collect(); + + (source_line, preceding_lines, following_lines) +} + impl ErrorContext { /// Create a new error context with a source line. pub fn new(source_line: impl Into) -> Self { Self { - source_line: source_line.into(), - preceding_lines: Vec::new(), - following_lines: Vec::new(), + source: ErrorContextSource::Inline { + source_line: source_line.into(), + preceding_lines: Vec::new(), + following_lines: Vec::new(), + indicator: None, + }, suggestion: None, - indicator: None, } } /// Add preceding context lines. pub fn with_preceding_lines(mut self, lines: Vec) -> Self { - self.preceding_lines = lines; + self.ensure_inline(); + if let ErrorContextSource::Inline { + preceding_lines, .. + } = &mut self.source + { + *preceding_lines = lines; + } self } /// Add following context lines. pub fn with_following_lines(mut self, lines: Vec) -> Self { - self.following_lines = lines; + self.ensure_inline(); + if let ErrorContextSource::Inline { + following_lines, .. + } = &mut self.source + { + *following_lines = lines; + } self } @@ -119,49 +216,72 @@ impl ErrorContext { /// Add a column indicator (caret) pointing to the error position. pub fn with_indicator(mut self, column: usize) -> Self { - let indicator = format!("{}^", " ".repeat(column)); - self.indicator = Some(indicator); + self.ensure_inline(); + if let ErrorContextSource::Inline { indicator, .. } = &mut self.source { + let indicator_value = format!("{}^", " ".repeat(column.saturating_sub(1))); + *indicator = Some(indicator_value); + } self } - /// Create error context from input string with automatic context - /// extraction. - pub fn from_input( - input: &str, + fn ensure_inline(&mut self) { + if let ErrorContextSource::Lazy { + input, + line, + column, + context_lines, + } = &self.source + { + let (source_line, preceding_lines, following_lines) = + extract_lines_to_strings(input, *line, *context_lines); + let indicator = if *column > 0 { + Some(format!("{}^", " ".repeat(column.saturating_sub(1)))) + } else { + None + }; + + self.source = ErrorContextSource::Inline { + source_line, + preceding_lines, + following_lines, + indicator, + }; + } + } + + /// Create error context from a shared input buffer with lazy extraction. + pub fn from_shared_input( + input: Arc, line: usize, column: usize, context_lines: usize, ) -> Option { - let lines: Vec<&str> = input.lines().collect(); - - if line == 0 || line > lines.len() { + let line_count = input.lines().count(); + if line == 0 || line > line_count { return None; } - let line_idx = line - 1; - let source_line = lines.get(line_idx)?.to_string(); - - let start_line = line_idx.saturating_sub(context_lines); - let end_line = (line_idx + context_lines + 1).min(lines.len()); - - let preceding_lines = lines[start_line..line_idx] - .iter() - .map(|s| s.to_string()) - .collect(); - - let following_lines = lines[(line_idx + 1)..end_line] - .iter() - .map(|s| s.to_string()) - .collect(); - Some(Self { - source_line, - preceding_lines, - following_lines, + source: ErrorContextSource::Lazy { + input, + line, + column, + context_lines, + }, suggestion: None, - indicator: Some(format!("{}^", " ".repeat(column.saturating_sub(1)))), }) } + + /// Create error context from input string with automatic context + /// extraction. + pub fn from_input( + input: &str, + line: usize, + column: usize, + context_lines: usize, + ) -> Option { + Self::from_shared_input(Arc::from(input), line, column, context_lines) + } } impl ToonError { @@ -285,9 +405,18 @@ mod tests { .with_suggestion("Try using quotes") .with_indicator(5); - assert_eq!(ctx.source_line, "test line"); + match &ctx.source { + ErrorContextSource::Inline { + source_line, + indicator, + .. + } => { + assert_eq!(source_line, "test line"); + assert!(indicator.is_some()); + } + ErrorContextSource::Lazy { .. } => panic!("Expected inline context"), + } assert_eq!(ctx.suggestion, Some("Try using quotes".to_string())); - assert!(ctx.indicator.is_some()); } #[test] @@ -297,9 +426,24 @@ mod tests { assert!(ctx.is_some()); let ctx = ctx.unwrap(); - assert_eq!(ctx.source_line, "line 2 with error"); - assert_eq!(ctx.preceding_lines, vec!["line 1"]); - assert_eq!(ctx.following_lines, vec!["line 3"]); + match &ctx.source { + ErrorContextSource::Lazy { + line, + column, + context_lines, + .. + } => { + assert_eq!(*line, 2); + assert_eq!(*column, 6); + assert_eq!(*context_lines, 1); + } + ErrorContextSource::Inline { .. } => panic!("Expected lazy context"), + } + + let rendered = format!("{ctx}"); + assert!(rendered.contains("line 2 with error")); + assert!(rendered.contains("line 1")); + assert!(rendered.contains("line 3")); } #[test] diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 1b300cb..64c702c 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -10,6 +10,9 @@ pub use string::{ escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, }; +#[cfg(feature = "parallel")] +use rayon::prelude::*; + use crate::types::{JsonValue as Value, Number}; /// Context for determining when quoting is needed. @@ -19,6 +22,9 @@ pub enum QuotingContext { ArrayValue, } +#[cfg(feature = "parallel")] +const PARALLEL_THRESHOLD: usize = 256; + /// Normalize a JSON value (converts NaN/Infinity to null, -0 to 0). pub fn normalize(value: Value) -> Value { match value { @@ -39,11 +45,31 @@ pub fn normalize(value: Value) -> Value { } } Value::Object(obj) => { + #[cfg(feature = "parallel")] + { + if obj.len() >= PARALLEL_THRESHOLD { + let entries: Vec<(String, Value)> = obj.into_iter().collect(); + let normalized_entries: Vec<(String, Value)> = entries + .into_par_iter() + .map(|(k, v)| (k, normalize(v))) + .collect(); + return Value::Object(IndexMap::from_iter(normalized_entries)); + } + } + let normalized: IndexMap = obj.into_iter().map(|(k, v)| (k, normalize(v))).collect(); Value::Object(normalized) } Value::Array(arr) => { + #[cfg(feature = "parallel")] + { + if arr.len() >= PARALLEL_THRESHOLD { + let normalized: Vec = arr.into_par_iter().map(normalize).collect(); + return Value::Array(normalized); + } + } + let normalized: Vec = arr.into_iter().map(normalize).collect(); Value::Array(normalized) } diff --git a/tests/errors.rs b/tests/errors.rs index 45fa38b..6df038e 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -447,11 +447,7 @@ fn test_error_context_information() { ToonError::ParseError { context: Some(ctx), .. } => { - println!( - "Error context has {} preceding lines, {} following lines", - ctx.preceding_lines.len(), - ctx.following_lines.len() - ); + println!("Parse error context:\n{ctx}"); } ToonError::LengthMismatch { context: Some(ctx), .. From f5b1b7eeaf84ec7e6d96057d2fe4b5d607829a96 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 18:01:33 +0000 Subject: [PATCH 15/24] fix: preserve large integer formatting --- src/utils/number.rs | 113 +++++++++++++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 38 deletions(-) diff --git a/src/utils/number.rs b/src/utils/number.rs index 0a62f78..bb16139 100644 --- a/src/utils/number.rs +++ b/src/utils/number.rs @@ -1,41 +1,60 @@ +use itoa::Buffer as ItoaBuffer; +use ryu::Buffer as RyuBuffer; + use crate::types::Number; /// Format a number in TOON canonical form (no exponents, no trailing zeros). pub fn format_canonical_number(n: &Number) -> String { - if let Some(i) = n.as_i64() { - return i.to_string(); - } + let mut out = String::new(); + write_canonical_number_into(n, &mut out); + out +} - if let Some(u) = n.as_u64() { - return u.to_string(); +pub fn write_canonical_number_into(n: &Number, out: &mut String) { + match n { + Number::PosInt(u) => write_u64(out, *u), + Number::NegInt(i) => write_i64(out, *i), + Number::Float(f) => write_f64_canonical_into(*f, out), } +} - if let Some(f) = n.as_f64() { - return format_f64_canonical(f); - } +fn write_u64(out: &mut String, value: u64) { + let mut buf = ItoaBuffer::new(); + out.push_str(buf.format(value)); +} - n.to_string() +fn write_i64(out: &mut String, value: i64) { + let mut buf = ItoaBuffer::new(); + out.push_str(buf.format(value)); } -fn format_f64_canonical(f: f64) -> String { +fn write_f64_canonical_into(f: f64, out: &mut String) { // Normalize integer-valued floats to integers if f.is_finite() && f.fract() == 0.0 && f.abs() <= i64::MAX as f64 { - return format!("{}", f as i64); + write_i64(out, f as i64); + return; } - let default_format = format!("{f}"); + if !f.is_finite() { + out.push('0'); + return; + } + + let mut buf = RyuBuffer::new(); + let formatted = buf.format(f); // Handle cases where Rust would use exponential notation - if default_format.contains('e') || default_format.contains('E') { - format_without_exponent(f) + if formatted.contains('e') || formatted.contains('E') { + write_without_exponent(f, out); } else { - remove_trailing_zeros(&default_format) + push_trimmed_decimal(formatted, out); } } -fn format_without_exponent(f: f64) -> String { +fn write_without_exponent(f: f64, out: &mut String) { if !f.is_finite() { - return "0".to_string(); + out.push('0'); + return; } if f.abs() >= 1.0 { @@ -44,42 +63,60 @@ fn format_without_exponent(f: f64) -> String { let frac_part = abs_f.fract(); if frac_part == 0.0 { - format!("{}{}", if f < 0.0 { "-" } else { "" }, int_part as i64) + if abs_f <= i64::MAX as f64 { + if f < 0.0 { + out.push('-'); + } + write_i64(out, int_part as i64); + } else { + let result = format!("{f:.0}"); + push_trimmed_decimal(&result, out); + } } else { // High precision to avoid exponent, then trim trailing zeros let result = format!("{f:.17}"); - remove_trailing_zeros(&result) + push_trimmed_decimal(&result, out); } } else if f == 0.0 { - "0".to_string() + out.push('0'); } else { // Small numbers: use high precision to avoid exponent - let result = format!("{f:.17}",); - remove_trailing_zeros(&result) + let result = format!("{f:.17}"); + push_trimmed_decimal(&result, out); } } +#[cfg(test)] fn remove_trailing_zeros(s: &str) -> String { - if !s.contains('.') { + if let Some((int_part, frac_part)) = s.split_once('.') { + let trimmed = frac_part.trim_end_matches('0'); + if trimmed.is_empty() { + int_part.to_string() + } else { + let mut out = String::with_capacity(int_part.len() + 1 + trimmed.len()); + out.push_str(int_part); + out.push('.'); + out.push_str(trimmed); + out + } + } else { // No decimal point, return as-is - return s.to_string(); - } - - let parts: Vec<&str> = s.split('.').collect(); - if parts.len() != 2 { - return s.to_string(); + s.to_string() } +} - let int_part = parts[0]; - let mut frac_part = parts[1].to_string(); - - frac_part = frac_part.trim_end_matches('0').to_string(); - - if frac_part.is_empty() { - // All zeros removed, return as integer - int_part.to_string() +fn push_trimmed_decimal(s: &str, out: &mut String) { + if let Some((int_part, frac_part)) = s.split_once('.') { + let trimmed = frac_part.trim_end_matches('0'); + if trimmed.is_empty() { + out.push_str(int_part); + } else { + out.push_str(int_part); + out.push('.'); + out.push_str(trimmed); + } } else { - format!("{int_part}.{frac_part}") + out.push_str(s); } } From 56d9f4b36db7e52f9d225f82b3d30461ac48ef88 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 18:12:25 +0000 Subject: [PATCH 16/24] docs: refresh performance snapshot --- Cargo.toml | 2 + README.md | 19 +++ src/constants.rs | 2 +- src/decode/expansion.rs | 24 ++-- src/decode/parser.rs | 32 +++-- src/decode/scanner.rs | 281 +++++++++++++++++++++++++++++++--------- src/encode/mod.rs | 177 +++++++++++++------------ src/encode/writer.rs | 60 +++++++-- src/types/folding.rs | 17 +-- src/utils/literal.rs | 24 ++-- src/utils/mod.rs | 5 +- src/utils/string.rs | 88 ++++++++----- 12 files changed, 490 insertions(+), 241 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5d3918f..108b955 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,8 @@ serde = { version = "1.0.228", features = ["derive"] } indexmap = "2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } thiserror = "2.0.17" +itoa = "1.0" +ryu = "1.0" rayon = { version = "1.10", optional = true } # CLI dependencies (gated behind "cli"/"cli-stats" features) diff --git a/README.md b/README.md index 2e7d1c6..96b2178 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,25 @@ users[2]{id,name}: - **Strict Validation**: Enforces all spec rules (configurable) - **Well-Tested**: Comprehensive test suite with unit tests, spec fixtures, and real-world scenarios +## Performance Snapshot (Criterion) + +Snapshot from commit `f5b1b7e` using: +`cargo bench --bench encode_decode -- --save-baseline current --noplot` + +| Benchmark | Median | +| --- | --- | +| `tabular/encode/128` | 145.81 us | +| `tabular/decode/128` | 115.51 us | +| `tabular/encode/1024` | 1.2059 ms | +| `tabular/decode/1024` | 949.65 us | +| `deep_object/encode/32` | 11.766 us | +| `deep_object/decode/32` | 10.930 us | +| `deep_object/encode/128` | 46.867 us | +| `deep_object/decode/128` | 49.468 us | +| `decode_long_unquoted` | 10.554 us | + +Numbers vary by machine; use Criterion baselines to compare before/after changes. + ## Installation ### As a Library diff --git a/src/constants.rs b/src/constants.rs index 2d9d444..9a57f2f 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -24,7 +24,7 @@ pub(crate) const QUOTED_KEY_MARKER: char = '\x00'; #[inline] pub fn is_structural_char(ch: char) -> bool { - STRUCTURAL_CHARS.contains(&ch) + matches!(ch, '[' | ']' | '{' | '}' | ':' | '-') } #[inline] diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index 4162d24..2162aa0 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -18,18 +18,24 @@ pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option return None; } - let segments: Vec<&str> = key.split('.').collect(); + let mut segment_count = 0; + for segment in key.split('.') { + if segment.is_empty() || !is_identifier_segment(segment) { + return None; + } + segment_count += 1; + } - if segments.len() < 2 { + if segment_count < 2 { return None; } - // Only expand if all segments are valid identifiers (safety requirement) - if segments.iter().all(|s| is_identifier_segment(s)) { - Some(segments) - } else { - None + let mut segments = Vec::with_capacity(segment_count); + for segment in key.split('.') { + segments.push(segment); } + + Some(segments) } } } @@ -111,7 +117,9 @@ pub fn expand_paths_in_object( None => { // Strip marker from quoted keys let clean_key = if key.starts_with(QUOTED_KEY_MARKER) { - key.strip_prefix(QUOTED_KEY_MARKER).unwrap().to_string() + let mut cleaned = key; + cleaned.remove(0); + cleaned } else { key }; diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 795e9b3..80f23c4 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -490,7 +490,8 @@ impl Parser { if matches!(self.current_token, Token::Newline | Token::Eof) { let has_children = if matches!(self.current_token, Token::Newline) { let current_depth_indent = self.options.indent.get_spaces() * (depth + 1); - let next_indent = self.normalize_indent(self.scanner.count_leading_spaces()); + let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self.normalize_indent(next_indent); next_indent >= current_depth_indent } else { false @@ -505,7 +506,7 @@ impl Parser { self.parse_value_with_depth(depth + 1) } else { // Check if there's more content after the current token - let (rest, leading_space) = self.scanner.read_rest_of_line_with_space_info(); + let (rest, leading_space) = self.scanner.read_rest_of_line_with_space_count(); let result = if rest.is_empty() { // Single token - convert directly to avoid redundant parsing @@ -542,14 +543,13 @@ impl Parser { Token::Null => 4, _ => 0, }; - let mut value_str = - String::with_capacity(token_len + leading_space.len() + rest.len()); + let mut value_str = String::with_capacity(token_len + leading_space + rest.len()); match &self.current_token { Token::String(s, true) => { // Quoted strings need quotes preserved for re-parsing value_str.push('"'); - value_str.push_str(&crate::utils::escape_string(s)); + crate::utils::escape_string_into(&mut value_str, s); value_str.push('"'); } Token::String(s, false) => value_str.push_str(s), @@ -563,8 +563,8 @@ impl Parser { } // Only add space if there was whitespace in the original input - if !rest.is_empty() { - value_str.push_str(&leading_space); + if !rest.is_empty() && leading_space > 0 { + value_str.extend(std::iter::repeat_n(' ', leading_space)); } value_str.push_str(&rest); @@ -1152,8 +1152,8 @@ impl Parser { // Check if there are more fields at the same indentation level let should_parse_more_fields = if matches!(self.current_token, Token::Newline) { - let next_indent = self - .normalize_indent(self.scanner.count_leading_spaces()); + let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self.normalize_indent(next_indent); if next_indent < field_indent { false @@ -1218,9 +1218,8 @@ impl Parser { obj.insert(field_key, field_value); if matches!(self.current_token, Token::Newline) { - let next_indent = self.normalize_indent( - self.scanner.count_leading_spaces(), - ); + let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self.normalize_indent(next_indent); if next_indent < field_indent { break; } @@ -1346,8 +1345,8 @@ impl Parser { let should_parse_more_fields = if matches!(self.current_token, Token::Newline) { - let next_indent = self - .normalize_indent(self.scanner.count_leading_spaces()); + let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self.normalize_indent(next_indent); if next_indent < field_indent { false @@ -1405,9 +1404,8 @@ impl Parser { obj.insert(field_key, field_value); if matches!(self.current_token, Token::Newline) { - let next_indent = self.normalize_indent( - self.scanner.count_leading_spaces(), - ); + let next_indent = self.scanner.count_leading_spaces(); + let next_indent = self.normalize_indent(next_indent); if next_indent < field_indent { break; } diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 61cede4..0c61996 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -32,11 +32,19 @@ pub struct Scanner { column: usize, active_delimiter: Option, last_line_indent: usize, + cached_indent: Option, coerce_types: bool, indent_width: usize, allow_tab_indent: bool, } +#[derive(Clone, Copy, Debug)] +struct CachedIndent { + position: usize, + indent: usize, + chars: usize, +} + impl Scanner { /// Create a new scanner for the given input string. pub fn new(input: &str) -> Self { @@ -51,6 +59,7 @@ impl Scanner { column: 1, active_delimiter: None, last_line_indent: 0, + cached_indent: None, coerce_types: true, indent_width: DEFAULT_INDENT, allow_tab_indent: false, @@ -85,11 +94,16 @@ impl Scanner { } pub fn peek(&self) -> Option { - self.input[self.position..].chars().next() + let bytes = self.input.as_bytes(); + match bytes.get(self.position) { + Some(&byte) if byte.is_ascii() => Some(byte as char), + Some(_) => self.input[self.position..].chars().next(), + None => None, + } } - pub fn count_leading_spaces(&self) -> usize { - self.count_indent_from(self.position) + pub fn count_leading_spaces(&mut self) -> usize { + self.peek_indent() } pub fn count_spaces_after_newline(&self) -> usize { @@ -102,21 +116,51 @@ impl Scanner { } pub fn peek_ahead(&self, offset: usize) -> Option { - self.input[self.position..].chars().nth(offset) + let bytes = self.input.as_bytes(); + let mut idx = self.position; + let mut remaining = offset; + + while let Some(&byte) = bytes.get(idx) { + if byte.is_ascii() { + if remaining == 0 { + return Some(byte as char); + } + idx += 1; + remaining -= 1; + continue; + } + + return self.input[self.position..].chars().nth(offset); + } + + None } pub fn advance(&mut self) -> Option { - if let Some(ch) = self.peek() { - self.position += ch.len_utf8(); - if ch == '\n' { - self.line += 1; - self.column = 1; - } else { - self.column += 1; + let bytes = self.input.as_bytes(); + match bytes.get(self.position) { + Some(&byte) if byte.is_ascii() => { + self.position += 1; + if byte == b'\n' { + self.line += 1; + self.column = 1; + } else { + self.column += 1; + } + Some(byte as char) } - Some(ch) - } else { - None + Some(_) => { + let ch = self.input[self.position..].chars().next()?; + self.position += ch.len_utf8(); + if ch == '\n' { + self.line += 1; + self.column = 1; + } else { + self.column += 1; + } + Some(ch) + } + None => None, } } @@ -131,50 +175,96 @@ impl Scanner { } fn count_indent_from(&self, mut idx: usize) -> usize { + self.count_indent_from_with_chars(&mut idx).0 + } + + fn count_indent_from_with_chars(&self, idx: &mut usize) -> (usize, usize) { let mut count = 0; + let mut chars = 0; let bytes = self.input.as_bytes(); - while idx < bytes.len() { - match bytes[idx] { + while *idx < bytes.len() { + match bytes[*idx] { b' ' => { count += 1; - idx += 1; + chars += 1; + *idx += 1; } b'\t' if self.allow_tab_indent => { count += self.indent_width; - idx += 1; + chars += 1; + *idx += 1; } _ => break, } } - count + (count, chars) + } + + fn peek_indent(&mut self) -> usize { + if let Some(cached) = self.cached_indent { + if cached.position == self.position { + return cached.indent; + } + } + + let mut idx = self.position; + let (indent, chars) = self.count_indent_from_with_chars(&mut idx); + self.cached_indent = Some(CachedIndent { + position: self.position, + indent, + chars, + }); + indent } /// Scan the next token from the input. pub fn scan_token(&mut self) -> ToonResult { if self.column == 1 { - let mut count = 0; - while let Some(ch) = self.peek() { - match ch { - ' ' => { - count += 1; - self.advance(); + let mut indent_consumed = false; + if let Some(cached) = self.cached_indent.take() { + if cached.position == self.position { + self.position += cached.chars; + self.column += cached.chars; + if !self.allow_tab_indent && matches!(self.peek(), Some('\t')) { + let (line, col) = self.current_position(); + return Err(ToonError::parse_error( + line, + col, + "Tabs are not allowed in indentation", + )); } - '\t' => { - if !self.allow_tab_indent { - let (line, col) = self.current_position(); - return Err(ToonError::parse_error( - line, - col + count, - "Tabs are not allowed in indentation", - )); + self.last_line_indent = cached.indent; + indent_consumed = true; + } else { + self.cached_indent = Some(cached); + } + } + + if !indent_consumed { + let mut count = 0; + while let Some(ch) = self.peek() { + match ch { + ' ' => { + count += 1; + self.advance(); + } + '\t' => { + if !self.allow_tab_indent { + let (line, col) = self.current_position(); + return Err(ToonError::parse_error( + line, + col + count, + "Tabs are not allowed in indentation", + )); + } + count += self.indent_width; + self.advance(); } - count += self.indent_width; - self.advance(); + _ => break, } - _ => break, } + self.last_line_indent = count; } - self.last_line_indent = count; } self.skip_whitespace(); @@ -287,30 +377,44 @@ impl Scanner { fn scan_unquoted_string(&mut self) -> ToonResult { let mut value = String::new(); + let bytes = self.input.as_bytes(); + let mut idx = self.position; + let mut start = idx; - while let Some(ch) = self.peek() { - if ch == '\n' - || ch == ' ' - || ch == ':' - || ch == '[' - || ch == ']' - || ch == '{' - || ch == '}' - { - break; + while idx < bytes.len() { + let byte = bytes[idx]; + if byte.is_ascii() { + let ch = byte as char; + if self.is_unquoted_terminator(ch) { + break; + } + idx += 1; + continue; + } + + if idx > start { + value.push_str(&self.input[start..idx]); + self.position = idx; + self.column += idx - start; } - // Active delimiters stop the string; otherwise they're part of it - if let Some(active) = self.active_delimiter { - if (active == Delimiter::Comma && ch == ',') - || (active == Delimiter::Pipe && ch == '|') - || (active == Delimiter::Tab && ch == '\t') - { + while let Some(ch) = self.peek() { + if self.is_unquoted_terminator(ch) { break; } + value.push(ch); + self.advance(); } - value.push(ch); - self.advance(); + + start = self.position; + idx = self.position; + break; + } + + if idx > start { + value.push_str(&self.input[start..idx]); + self.position = idx; + self.column += idx - start; } // Single-char delimiters kept as-is, others trimmed @@ -331,16 +435,30 @@ impl Scanner { } } + fn is_unquoted_terminator(&self, ch: char) -> bool { + if matches!(ch, '\n' | ' ' | ':' | '[' | ']' | '{' | '}') { + return true; + } + + if let Some(active) = self.active_delimiter { + return matches!( + (active, ch), + (Delimiter::Comma, ',') | (Delimiter::Pipe, '|') | (Delimiter::Tab, '\t') + ); + } + + false + } + pub fn get_last_line_indent(&self) -> usize { self.last_line_indent } fn scan_number_string(&mut self, negative: bool) -> ToonResult { - let mut num_str = if negative { - String::from("-") - } else { - String::new() - }; + let mut num_str = String::with_capacity(32); + if negative { + num_str.push('-'); + } while let Some(ch) = self.peek() { if ch.is_ascii_digit() || ch == '.' || ch == 'e' || ch == 'E' || ch == '+' || ch == '-' @@ -413,9 +531,19 @@ impl Scanner { /// Returns the content and any leading spaces between the current token /// and the rest of the line. pub fn read_rest_of_line_with_space_info(&mut self) -> (String, String) { - let mut leading_space = String::new(); + let (content, leading_space) = self.read_rest_of_line_with_space_count(); + let mut spaces = String::with_capacity(leading_space); + spaces.extend(std::iter::repeat_n(' ', leading_space)); + (content, spaces) + } + + /// Read the rest of the current line (until newline or EOF). + /// Returns the content and number of leading spaces between the current + /// token and the rest of the line. + pub fn read_rest_of_line_with_space_count(&mut self) -> (String, usize) { + let mut leading_space = 0usize; while matches!(self.peek(), Some(' ')) { - leading_space.push(' '); + leading_space += 1; self.advance(); } @@ -428,12 +556,14 @@ impl Scanner { self.advance(); } - (result.trim_end().to_string(), leading_space) + let trimmed_len = result.trim_end().len(); + result.truncate(trimmed_len); + (result, leading_space) } /// Read the rest of the current line (until newline or EOF). pub fn read_rest_of_line(&mut self) -> String { - self.read_rest_of_line_with_space_info().0 + self.read_rest_of_line_with_space_count().0 } /// Parse a complete value string into a token. @@ -670,6 +800,29 @@ mod tests { assert!(leading_space.is_empty()); } + #[test] + fn test_read_rest_of_line_with_space_count() { + let mut scanner = Scanner::new(" world"); + let (content, leading_space) = scanner.read_rest_of_line_with_space_count(); + assert_eq!(content, "world"); + assert_eq!(leading_space, 1); + + let mut scanner = Scanner::new("world"); + let (content, leading_space) = scanner.read_rest_of_line_with_space_count(); + assert_eq!(content, "world"); + assert_eq!(leading_space, 0); + + let mut scanner = Scanner::new("(hello)"); + let (content, leading_space) = scanner.read_rest_of_line_with_space_count(); + assert_eq!(content, "(hello)"); + assert_eq!(leading_space, 0); + + let mut scanner = Scanner::new(""); + let (content, leading_space) = scanner.read_rest_of_line_with_space_count(); + assert_eq!(content, ""); + assert_eq!(leading_space, 0); + } + #[test] fn test_parse_value_string() { let scanner = Scanner::new(""); diff --git a/src/encode/mod.rs b/src/encode/mod.rs index d812748..3943903 100644 --- a/src/encode/mod.rs +++ b/src/encode/mod.rs @@ -10,7 +10,7 @@ use crate::{ types::{ EncodeOptions, IntoJsonValue, JsonValue as Value, KeyFoldingMode, ToonError, ToonResult, }, - utils::{format_canonical_number, normalize, validation::validate_depth, QuotingContext}, + utils::{normalize, validation::validate_depth, QuotingContext}, }; /// Encode any serializable value to TOON format. @@ -209,42 +209,45 @@ fn write_object_impl( ) -> ToonResult<()> { validate_depth(depth, MAX_DEPTH)?; - let keys: Vec<&String> = obj.keys().collect(); - let mut key_set: HashSet<&str> = HashSet::with_capacity(keys.len()); - let mut prefix_conflicts: HashSet<&str> = HashSet::new(); - - for key in &keys { - key_set.insert(key.as_str()); - if key.contains('.') { - let mut start = 0; - while let Some(pos) = key[start..].find('.') { - let end = start + pos; - if end > 0 { - prefix_conflicts.insert(&key[..end]); + let allow_folding = !disable_folding && writer.options.key_folding == KeyFoldingMode::Safe; + let (key_set, prefix_conflicts) = if allow_folding { + let mut key_set: HashSet<&str> = HashSet::with_capacity(obj.len()); + let mut prefix_conflicts: HashSet<&str> = HashSet::new(); + + for key in obj.keys() { + let key_str = key.as_str(); + key_set.insert(key_str); + if key_str.contains('.') { + let mut start = 0; + while let Some(pos) = key_str[start..].find('.') { + let end = start + pos; + if end > 0 { + prefix_conflicts.insert(&key_str[..end]); + } + start = end + 1; } - start = end + 1; } } - } - for (i, key) in keys.iter().enumerate() { + (key_set, prefix_conflicts) + } else { + (HashSet::new(), HashSet::new()) + }; + + for (i, (key, value)) in obj.iter().enumerate() { if i > 0 { writer.write_newline()?; } - let key = *key; let key_str = key.as_str(); - let value = obj.get(key).expect("key exists in field list"); // Check if this key-value pair can be folded (v1.5 feature) // Don't fold if any sibling key is a dotted path starting with this key // (e.g., don't fold inside "data" if "data.meta.items" exists as a sibling) - let has_conflicting_sibling = key_str.contains('.') || prefix_conflicts.contains(key_str); + let has_conflicting_sibling = + allow_folding && (key_str.contains('.') || prefix_conflicts.contains(key_str)); - let folded = if !disable_folding - && writer.options.key_folding == KeyFoldingMode::Safe - && !has_conflicting_sibling - { + let folded = if allow_folding && !has_conflicting_sibling { folding::analyze_foldable_chain(key_str, value, writer.options.flatten_depth, &key_set) } else { None @@ -333,63 +336,64 @@ fn write_array( // Select format based on array content: tabular (uniform objects) > inline // primitives > nested list - if let Some(keys) = is_tabular_array(arr) { - encode_tabular_array(writer, key, arr, &keys, depth)?; - } else if is_primitive_array(arr) { - encode_primitive_array(writer, key, arr, depth)?; - } else { - encode_nested_array(writer, key, arr, depth)?; + match classify_array(arr) { + ArrayKind::Tabular(keys) => encode_tabular_array(writer, key, arr, &keys, depth)?, + ArrayKind::Primitive => encode_primitive_array(writer, key, arr, depth)?, + ArrayKind::Nested => encode_nested_array(writer, key, arr, depth)?, } Ok(()) } -/// Check if an array can be encoded as tabular format (uniform objects with -/// primitive values). -fn is_tabular_array(arr: &[Value]) -> Option> { - if arr.is_empty() { - return None; - } - - let first = arr.first()?; - if !first.is_object() { - return None; - } +enum ArrayKind<'a> { + Tabular(Vec<&'a str>), + Primitive, + Nested, +} - let first_obj = first.as_object()?; - let keys: Vec = first_obj.keys().cloned().collect(); +/// Classify array shape for encoding (tabular, primitive, nested). +fn classify_array<'a>(arr: &'a [Value]) -> ArrayKind<'a> { + let first = match arr.first() { + Some(value) => value, + None => return ArrayKind::Primitive, + }; - // First object must have only primitive values - for value in first_obj.values() { - if !is_primitive(value) { - return None; + if let Value::Object(first_obj) = first { + if !first_obj.values().all(is_primitive) { + return ArrayKind::Nested; } - } - // All remaining objects must match: same keys and all primitive values - for val in arr.iter().skip(1) { - if let Some(obj) = val.as_object() { + let keys: Vec<&str> = first_obj.keys().map(|key| key.as_str()).collect(); + + for val in arr.iter().skip(1) { + let obj = match val.as_object() { + Some(obj) => obj, + None => return ArrayKind::Nested, + }; + if obj.len() != keys.len() { - return None; + return ArrayKind::Nested; } - // Verify all keys from first object exist (order doesn't matter) + for key in &keys { - if !obj.contains_key(key) { - return None; + if !obj.contains_key(*key) { + return ArrayKind::Nested; } } - // All values must be primitives - for value in obj.values() { - if !is_primitive(value) { - return None; - } + + if !obj.values().all(is_primitive) { + return ArrayKind::Nested; } - } else { - return None; } + + return ArrayKind::Tabular(keys); } - Some(keys) + if arr.iter().all(is_primitive) { + ArrayKind::Primitive + } else { + ArrayKind::Nested + } } /// Check if a value is a primitive (not array or object). @@ -401,10 +405,6 @@ fn is_primitive(value: &Value) -> bool { } /// Check if all array elements are primitives. -fn is_primitive_array(arr: &[Value]) -> bool { - arr.iter().all(is_primitive) -} - fn encode_primitive_array( writer: &mut writer::Writer, key: Option<&str>, @@ -434,11 +434,10 @@ fn write_primitive_value( ) -> ToonResult<()> { match value { Value::Null => writer.write_str("null"), - Value::Bool(b) => writer.write_str(&b.to_string()), + Value::Bool(b) => writer.write_str(if *b { "true" } else { "false" }), Value::Number(n) => { // Format in canonical TOON form (no exponents, no trailing zeros) - let num_str = format_canonical_number(n); - writer.write_str(&num_str) + writer.write_canonical_number(n) } Value::String(s) => { if writer.needs_quoting(s, context) { @@ -457,7 +456,7 @@ fn encode_tabular_array( writer: &mut writer::Writer, key: Option<&str>, arr: &[Value], - keys: &[String], + keys: &[&str], depth: usize, ) -> ToonResult<()> { writer.write_array_header(key, arr.len(), Some(keys), depth)?; @@ -476,7 +475,7 @@ fn encode_tabular_array( } // Missing fields become null - if let Some(val) = obj.get(key) { + if let Some(val) = obj.get(*key) { write_primitive_value(writer, val, QuotingContext::ArrayValue)?; } else { writer.write_str("null")?; @@ -502,12 +501,12 @@ fn encode_tabular_array( fn encode_list_item_tabular_array( writer: &mut writer::Writer, arr: &[Value], - keys: &[String], + keys: &[&str], depth: usize, ) -> ToonResult<()> { // Write array header without key (key already written on hyphen line) writer.write_char('[')?; - writer.write_str(&arr.len().to_string())?; + writer.write_usize(arr.len())?; if writer.options.delimiter != crate::types::Delimiter::Comma { writer.write_char(writer.options.delimiter.as_char())?; @@ -541,7 +540,7 @@ fn encode_list_item_tabular_array( } // Missing fields become null - if let Some(val) = obj.get(key) { + if let Some(val) = obj.get(*key) { write_primitive_value(writer, val, QuotingContext::ArrayValue)?; } else { writer.write_str("null")?; @@ -592,14 +591,30 @@ fn encode_nested_array( // (depth +2 relative to hyphen) for their nested content // (rows for tabular, items for non-uniform) writer.write_key(first_key)?; - - if let Some(keys) = is_tabular_array(arr) { - // Tabular array: write inline with correct indentation - encode_list_item_tabular_array(writer, arr, &keys, depth + 1)?; + if arr.is_empty() { + writer.write_empty_array_with_key(None, depth + 2)?; } else { - // Non-tabular array: write with depth offset - // (items at depth +2 instead of depth +1) - write_array(writer, None, arr, depth + 2)?; + match classify_array(arr) { + ArrayKind::Tabular(keys) => { + // Tabular array: write inline with correct indentation + encode_list_item_tabular_array( + writer, + arr, + &keys, + depth + 1, + )?; + } + ArrayKind::Primitive => { + // Non-tabular array: write with depth offset + // (items at depth +2 instead of depth +1) + encode_primitive_array(writer, None, arr, depth + 2)?; + } + ArrayKind::Nested => { + // Non-tabular array: write with depth offset + // (items at depth +2 instead of depth +1) + encode_nested_array(writer, None, arr, depth + 2)?; + } + } } } Value::Object(nested_obj) => { diff --git a/src/encode/writer.rs b/src/encode/writer.rs index 7edf35e..ae11ec3 100644 --- a/src/encode/writer.rs +++ b/src/encode/writer.rs @@ -1,7 +1,8 @@ use crate::{ - types::{Delimiter, EncodeOptions, ToonResult}, + types::{Delimiter, EncodeOptions, Number, ToonResult}, utils::{ - string::{is_valid_unquoted_key, needs_quoting, quote_string}, + number::write_canonical_number_into, + string::{escape_string_into, is_valid_unquoted_key, needs_quoting}, QuotingContext, }, }; @@ -11,15 +12,20 @@ pub struct Writer { buffer: String, pub(crate) options: EncodeOptions, active_delimiters: Vec, + indent_unit: String, + indent_cache: Vec, } impl Writer { /// Create a new writer with the given options. pub fn new(options: EncodeOptions) -> Self { + let indent_unit = " ".repeat(options.indent.get_spaces()); Self { buffer: String::new(), active_delimiters: vec![options.delimiter], options, + indent_unit, + indent_cache: vec![String::new()], } } @@ -44,10 +50,13 @@ impl Writer { } pub fn write_indent(&mut self, depth: usize) -> ToonResult<()> { - let indent_string = self.options.indent.get_string(depth); - if !indent_string.is_empty() { - self.buffer.push_str(&indent_string); + if depth == 0 || self.indent_unit.is_empty() { + return Ok(()); } + if depth >= self.indent_cache.len() { + self.extend_indent_cache(depth); + } + self.buffer.push_str(&self.indent_cache[depth]); Ok(()) } @@ -69,7 +78,7 @@ impl Writer { &mut self, key: Option<&str>, length: usize, - fields: Option<&[String]>, + fields: Option<&[&str]>, depth: usize, ) -> ToonResult<()> { if let Some(k) = key { @@ -80,7 +89,7 @@ impl Writer { } self.write_char('[')?; - self.write_str(&length.to_string())?; + self.write_usize(length)?; // Only write delimiter in header if it's not comma (comma is default/implied) if self.options.delimiter != Delimiter::Comma { @@ -117,7 +126,7 @@ impl Writer { self.write_key(k)?; } self.write_char('[')?; - self.write_str("0")?; + self.write_usize(0)?; if self.options.delimiter != Delimiter::Comma { self.write_delimiter()?; @@ -137,7 +146,10 @@ impl Writer { } pub fn write_quoted_string(&mut self, s: &str) -> ToonResult<()> { - self.write_str("e_string(s)) + self.buffer.push('"'); + escape_string_into(&mut self.buffer, s); + self.buffer.push('"'); + Ok(()) } pub fn write_value(&mut self, s: &str, context: QuotingContext) -> ToonResult<()> { @@ -148,6 +160,17 @@ impl Writer { } } + pub fn write_canonical_number(&mut self, n: &Number) -> ToonResult<()> { + write_canonical_number_into(n, &mut self.buffer); + Ok(()) + } + + pub fn write_usize(&mut self, value: usize) -> ToonResult<()> { + let mut buf = itoa::Buffer::new(); + self.buffer.push_str(buf.format(value as u64)); + Ok(()) + } + /// Push a new delimiter onto the stack (for nested arrays with different /// delimiters). pub fn push_active_delimiter(&mut self, delim: Delimiter) { @@ -169,6 +192,21 @@ impl Writer { fn get_document_delimiter_char(&self) -> char { self.options.delimiter.as_char() } + + fn extend_indent_cache(&mut self, depth: usize) { + while self.indent_cache.len() <= depth { + let next = match self.indent_cache.last() { + Some(prev) => { + let mut s = String::with_capacity(prev.len() + self.indent_unit.len()); + s.push_str(prev); + s.push_str(&self.indent_unit); + s + } + None => String::new(), + }; + self.indent_cache.push(next); + } + } } #[cfg(test)] @@ -239,7 +277,7 @@ mod tests { let opts = EncodeOptions::default(); let mut writer = Writer::new(opts); - let fields = vec!["id".to_string(), "name".to_string()]; + let fields = vec!["id", "name"]; writer .write_array_header(Some("users"), 2, Some(&fields), 0) @@ -259,7 +297,7 @@ mod tests { let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe); let mut writer = Writer::new(opts); - let fields = vec!["id".to_string(), "name".to_string()]; + let fields = vec!["id", "name"]; writer .write_array_header(Some("users"), 2, Some(&fields), 0) diff --git a/src/types/folding.rs b/src/types/folding.rs index ce968f1..04c4194 100644 --- a/src/types/folding.rs +++ b/src/types/folding.rs @@ -19,24 +19,21 @@ pub enum PathExpansionMode { /// Check if a key segment is a valid IdentifierSegment (stricter than unquoted /// keys). pub fn is_identifier_segment(s: &str) -> bool { - if s.is_empty() { + let bytes = s.as_bytes(); + if bytes.is_empty() { return false; } - let mut chars = s.chars(); - // First character must be letter or underscore - let first = match chars.next() { - Some(c) => c, - None => return false, - }; - - if !first.is_ascii_alphabetic() && first != '_' { + let first = bytes[0]; + if !first.is_ascii_alphabetic() && first != b'_' { return false; } // Remaining characters: letters, digits, or underscore (NO dots) - chars.all(|c| c.is_ascii_alphanumeric() || c == '_') + bytes[1..] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_') } #[cfg(test)] diff --git a/src/utils/literal.rs b/src/utils/literal.rs index 3f8aeea..7cf2c7c 100644 --- a/src/utils/literal.rs +++ b/src/utils/literal.rs @@ -17,34 +17,32 @@ pub fn is_structural_char(ch: char) -> bool { /// Check if a string looks like a number (starts with digit, no leading zeros). pub fn is_numeric_like(s: &str) -> bool { - if s.is_empty() { + let bytes = s.as_bytes(); + if bytes.is_empty() { return false; } - let chars: Vec = s.chars().collect(); let mut i = 0; - - if chars[i] == '-' { - i += 1; + if bytes[0] == b'-' { + i = 1; } - if i >= chars.len() { + if i >= bytes.len() { return false; } - if !chars[i].is_ascii_digit() { + let first = bytes[i]; + if !first.is_ascii_digit() { return false; } - if chars[i] == '0' && i + 1 < chars.len() && chars[i + 1].is_ascii_digit() { + if first == b'0' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() { return false; } - let has_valid_chars = chars[i..].iter().all(|c| { - c.is_ascii_digit() || *c == '.' || *c == 'e' || *c == 'E' || *c == '+' || *c == '-' - }); - - has_valid_chars + bytes[i..].iter().all(|b| { + b.is_ascii_digit() || *b == b'.' || *b == b'e' || *b == b'E' || *b == b'+' || *b == b'-' + }) } #[cfg(test)] diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 64c702c..d7d3e54 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,9 +5,10 @@ pub mod validation; use indexmap::IndexMap; pub use literal::{is_keyword, is_literal_like, is_numeric_like, is_structural_char}; -pub use number::format_canonical_number; +pub use number::{format_canonical_number, write_canonical_number_into}; pub use string::{ - escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, + escape_string, escape_string_into, is_valid_unquoted_key, needs_quoting, quote_string, + unescape_string, }; #[cfg(feature = "parallel")] diff --git a/src/utils/string.rs b/src/utils/string.rs index 8924442..3d1e630 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -4,18 +4,23 @@ use crate::{types::Delimiter, utils::literal}; pub fn escape_string(s: &str) -> String { let mut result = String::with_capacity(s.len()); + escape_string_into(&mut result, s); + + result +} + +/// Escape special characters in a string and append to the output buffer. +pub fn escape_string_into(out: &mut String, s: &str) { for ch in s.chars() { match ch { - '\n' => result.push_str("\\n"), - '\r' => result.push_str("\\r"), - '\t' => result.push_str("\\t"), - '"' => result.push_str("\\\""), - '\\' => result.push_str("\\\\"), - _ => result.push(ch), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + _ => out.push(ch), } } - - result } /// Unescape special characters in a quoted string. @@ -90,22 +95,19 @@ pub fn unescape_string(s: &str) -> Result { } fn is_valid_unquoted_key_internal(key: &str, allow_hyphen: bool) -> bool { - if key.is_empty() { + let bytes = key.as_bytes(); + if bytes.is_empty() { return false; } - let mut chars = key.chars(); - let first = if let Some(c) = chars.next() { - c - } else { - return false; - }; - - if !first.is_ascii_alphabetic() && first != '_' { + let first = bytes[0]; + if !first.is_ascii_alphabetic() && first != b'_' { return false; } - chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.' || (allow_hyphen && c == '-')) + bytes[1..].iter().all(|b| { + b.is_ascii_alphanumeric() || *b == b'_' || *b == b'.' || (allow_hyphen && *b == b'-') + }) } /// Check if a key can be written without quotes (alphanumeric, underscore, @@ -124,33 +126,47 @@ pub fn needs_quoting(s: &str, delimiter: char) -> bool { return true; } - if s.chars().any(literal::is_structural_char) { - return true; - } - - if s.contains('\\') || s.contains('"') { - return true; - } + let mut chars = s.chars(); + let first = match chars.next() { + Some(ch) => ch, + None => return true, + }; - if s.contains(delimiter) { + if first.is_whitespace() || first == '-' { return true; } - if s.contains('\n') || s.contains('\r') || s.contains('\t') { + if first == '\\' + || first == '"' + || first == delimiter + || first == '\n' + || first == '\r' + || first == '\t' + || literal::is_structural_char(first) + { return true; } - if s.starts_with(char::is_whitespace) || s.ends_with(char::is_whitespace) { + if first == '0' && chars.clone().next().is_some_and(|c| c.is_ascii_digit()) { return true; } - if s.starts_with('-') { - return true; + let mut last = first; + for ch in chars { + if literal::is_structural_char(ch) + || ch == '\\' + || ch == '"' + || ch == delimiter + || ch == '\n' + || ch == '\r' + || ch == '\t' + { + return true; + } + last = ch; } - // Check for leading zeros (e.g., "05", "007", "0123") - // Numbers with leading zeros must be quoted - if s.starts_with('0') && s.len() > 1 && s.chars().nth(1).is_some_and(|c| c.is_ascii_digit()) { + if last.is_whitespace() { return true; } @@ -159,7 +175,11 @@ pub fn needs_quoting(s: &str, delimiter: char) -> bool { /// Quote and escape a string. pub fn quote_string(s: &str) -> String { - format!("\"{}\"", escape_string(s)) + let mut result = String::with_capacity(s.len() + 2); + result.push('"'); + escape_string_into(&mut result, s); + result.push('"'); + result } pub fn split_by_delimiter(s: &str, delimiter: Delimiter) -> Vec { From 64b8e884d1df584ef70007a0940b898b8f86e00e Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 18:22:10 +0000 Subject: [PATCH 17/24] test: add fuzzing and boundary coverage --- fuzz/Cargo.toml | 28 ++++++++++++++++++++ fuzz/fuzz_targets/decode.rs | 13 +++++++++ fuzz/fuzz_targets/encode.rs | 11 ++++++++ tests/boundary.rs | 53 +++++++++++++++++++++++++++++++++++++ tests/spec_edge_cases.rs | 7 +++++ 5 files changed, 112 insertions(+) create mode 100644 fuzz/Cargo.toml create mode 100644 fuzz/fuzz_targets/decode.rs create mode 100644 fuzz/fuzz_targets/encode.rs create mode 100644 tests/boundary.rs diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..3bc837f --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "toon-format-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +toon-format = { path = ".." } +arbitrary = { version = "1", features = ["derive"] } +serde_json = "1" + +[[bin]] +name = "fuzz_decode" +path = "fuzz_targets/decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "fuzz_encode" +path = "fuzz_targets/encode.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/decode.rs b/fuzz/fuzz_targets/decode.rs new file mode 100644 index 0000000..ec4b24e --- /dev/null +++ b/fuzz/fuzz_targets/decode.rs @@ -0,0 +1,13 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use toon_format::{decode, DecodeOptions}; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + let _ = decode::(s, &DecodeOptions::default()); + + let strict = DecodeOptions::new().with_strict(true); + let _ = decode::(s, &strict); + } +}); diff --git a/fuzz/fuzz_targets/encode.rs b/fuzz/fuzz_targets/encode.rs new file mode 100644 index 0000000..003e429 --- /dev/null +++ b/fuzz/fuzz_targets/encode.rs @@ -0,0 +1,11 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use serde_json::Value; +use toon_format::{encode, EncodeOptions}; + +fuzz_target!(|data: &[u8]| { + if let Ok(json) = serde_json::from_slice::(data) { + let _ = encode(&json, &EncodeOptions::default()); + } +}); diff --git a/tests/boundary.rs b/tests/boundary.rs new file mode 100644 index 0000000..9803de7 --- /dev/null +++ b/tests/boundary.rs @@ -0,0 +1,53 @@ +//! Boundary condition tests + +use serde_json::json; +use toon_format::constants::MAX_DEPTH; +use toon_format::{decode, encode, DecodeOptions, EncodeOptions}; + +#[test] +fn test_max_depth_boundary() { + let mut nested = json!(null); + for _ in 0..=MAX_DEPTH { + nested = json!({"a": nested}); + } + + let encoded = encode(&nested, &EncodeOptions::default()); + assert!(encoded.is_ok()); + + let too_deep = json!({"a": nested}); + let result = encode(&too_deep, &EncodeOptions::default()); + assert!(result.is_err()); +} + +#[test] +fn test_large_array() { + let data: Vec = (0..10_000).collect(); + let large = json!(data); + let encoded = encode(&large, &EncodeOptions::default()).unwrap(); + let decoded: serde_json::Value = decode(&encoded, &DecodeOptions::default()).unwrap(); + assert_eq!(large, decoded); +} + +#[test] +fn test_very_long_string() { + let long_string = "x".repeat(100_000); + let value = json!({"data": long_string}); + let encoded = encode(&value, &EncodeOptions::default()).unwrap(); + let decoded: serde_json::Value = decode(&encoded, &DecodeOptions::default()).unwrap(); + assert_eq!(value, decoded); +} + +#[test] +fn test_empty_structures() { + let empty_obj = json!({}); + let empty_arr = json!([]); + + let encoded_obj = encode(&empty_obj, &EncodeOptions::default()).unwrap(); + let encoded_arr = encode(&empty_arr, &EncodeOptions::default()).unwrap(); + + let decoded_obj: serde_json::Value = decode(&encoded_obj, &DecodeOptions::default()).unwrap(); + let decoded_arr: serde_json::Value = decode(&encoded_arr, &DecodeOptions::default()).unwrap(); + + assert_eq!(empty_obj, decoded_obj); + assert_eq!(empty_arr, decoded_arr); +} diff --git a/tests/spec_edge_cases.rs b/tests/spec_edge_cases.rs index 9fb5a8a..d494562 100644 --- a/tests/spec_edge_cases.rs +++ b/tests/spec_edge_cases.rs @@ -33,6 +33,13 @@ fn test_negative_leading_zero_string() { assert_eq!(result, json!({"val": "-05"})); } +#[test] +fn test_field_list_delimiter_mismatch_strict() { + let input = "items[1|]{a,b}:\n 1,2"; + let opts = DecodeOptions::new().with_strict(true); + assert!(decode::(input, &opts).is_err()); +} + #[test] fn test_unquoted_tab_rejected_in_strict() { let input = "val: a\tb"; From 72e651b1bc7ae9068b1bd25dd7df7626d076d782 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 18:33:23 +0000 Subject: [PATCH 18/24] docs: document fuzzing workflow --- .gitignore | 5 +++++ README.md | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index 44d1841..b7d43f7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,11 @@ coverage/ tarpaulin-report.html cobertura.xml +# Fuzzing +fuzz/target/ +fuzz/corpus/ +fuzz/artifacts/ + # Rust target/ Cargo.lock diff --git a/README.md b/README.md index 96b2178..a86797e 100644 --- a/README.md +++ b/README.md @@ -612,6 +612,12 @@ cargo fmt # Build docs cargo doc --open + +# Fuzz targets (requires nightly + cargo-fuzz) +cargo install cargo-fuzz +cargo +nightly fuzz build +cargo +nightly fuzz run fuzz_decode -- -max_total_time=10 +cargo +nightly fuzz run fuzz_encode -- -max_total_time=10 ``` --- From 7c66bee359295b71de9cf6924b100b5e7a3f3839 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 18:47:17 +0000 Subject: [PATCH 19/24] test: expand helper and value coverage --- tests/decode_helpers.rs | 16 ++++ tests/encode_helpers.rs | 49 ++++++++++++ tests/error_context.rs | 101 +++++++++++++++++++++++++ tests/options.rs | 43 +++++++++++ tests/value.rs | 161 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 370 insertions(+) create mode 100644 tests/decode_helpers.rs create mode 100644 tests/encode_helpers.rs create mode 100644 tests/error_context.rs create mode 100644 tests/options.rs create mode 100644 tests/value.rs diff --git a/tests/decode_helpers.rs b/tests/decode_helpers.rs new file mode 100644 index 0000000..a2b2309 --- /dev/null +++ b/tests/decode_helpers.rs @@ -0,0 +1,16 @@ +use serde_json::{json, Value}; +use toon_format::{decode_no_coerce_with_options, decode_strict_with_options, DecodeOptions}; + +#[test] +fn test_decode_strict_with_options_forces_strict() { + let opts = DecodeOptions::new().with_strict(false); + let result: Result = decode_strict_with_options("items[2]: a", &opts); + assert!(result.is_err(), "Strict mode should reject length mismatch"); +} + +#[test] +fn test_decode_no_coerce_with_options_disables_coercion() { + let opts = DecodeOptions::new().with_coerce_types(true); + let result: Value = decode_no_coerce_with_options("value: 123", &opts).unwrap(); + assert_eq!(result, json!({"value": "123"})); +} diff --git a/tests/encode_helpers.rs b/tests/encode_helpers.rs new file mode 100644 index 0000000..6d3625c --- /dev/null +++ b/tests/encode_helpers.rs @@ -0,0 +1,49 @@ +use indexmap::IndexMap; +use serde_json::json; +use toon_format::{encode_array, encode_object, EncodeOptions, ToonError}; +use toon_format::types::{JsonValue, Number}; + +#[test] +fn test_encode_array_and_object_with_json() { + let array = json!(["a", "b"]); + let encoded = encode_array(&array, &EncodeOptions::default()).unwrap(); + assert!(encoded.starts_with("[2]:")); + + let object = json!({"a": 1}); + let encoded = encode_object(&object, &EncodeOptions::default()).unwrap(); + assert!(encoded.contains("a: 1")); +} + +#[test] +fn test_encode_array_object_type_mismatch() { + let err = encode_array(&json!({"a": 1}), &EncodeOptions::default()).unwrap_err(); + match err { + ToonError::TypeMismatch { expected, found } => { + assert_eq!(expected, "array"); + assert_eq!(found, "object"); + } + _ => panic!("Expected TypeMismatch for encode_array"), + } + + let err = encode_object(&json!(["a", "b"]), &EncodeOptions::default()).unwrap_err(); + match err { + ToonError::TypeMismatch { expected, found } => { + assert_eq!(expected, "object"); + assert_eq!(found, "array"); + } + _ => panic!("Expected TypeMismatch for encode_object"), + } +} + +#[test] +fn test_encode_array_object_with_json_value() { + let value = JsonValue::Array(vec![JsonValue::Number(Number::from(1))]); + let encoded = encode_array(value, &EncodeOptions::default()).unwrap(); + assert!(encoded.contains("1")); + + let mut obj = IndexMap::new(); + obj.insert("key".to_string(), JsonValue::Bool(true)); + let value = JsonValue::Object(obj); + let encoded = encode_object(value, &EncodeOptions::default()).unwrap(); + assert!(encoded.contains("key: true")); +} diff --git a/tests/error_context.rs b/tests/error_context.rs new file mode 100644 index 0000000..763e69b --- /dev/null +++ b/tests/error_context.rs @@ -0,0 +1,101 @@ +use std::sync::Arc; + +use toon_format::types::{ErrorContext, ToonError}; + +#[test] +fn test_error_context_inline_rendering() { + let ctx = ErrorContext::new("line 2") + .with_preceding_lines(vec!["line 1".to_string()]) + .with_following_lines(vec!["line 3".to_string()]) + .with_indicator(3) + .with_suggestion("use a colon"); + + let rendered = format!("{ctx}"); + assert!(rendered.contains("line 1")); + assert!(rendered.contains("> line 2")); + assert!(rendered.contains("^")); + assert!(rendered.contains("line 3")); + assert!(rendered.contains("Suggestion: use a colon")); +} + +#[test] +fn test_error_context_lazy_rendering_and_indicator() { + let input: Arc = Arc::from("one\ntwo\nthree"); + let ctx = ErrorContext::from_shared_input(Arc::clone(&input), 2, 2, 1).unwrap(); + + let rendered = format!("{ctx}"); + assert!(rendered.contains("> two")); + assert!(rendered.contains("^")); + assert!(rendered.contains("one")); + assert!(rendered.contains("three")); + + let inline = ctx.with_indicator(2); + let rendered_inline = format!("{inline}"); + assert!(rendered_inline.contains("^")); +} + +#[test] +fn test_error_context_invalid_line_returns_none() { + let input: Arc = Arc::from("line 1"); + assert!(ErrorContext::from_shared_input(Arc::clone(&input), 0, 1, 1).is_none()); + assert!(ErrorContext::from_shared_input(Arc::clone(&input), 2, 1, 1).is_none()); +} + +#[test] +fn test_toon_error_helpers() { + let err = ToonError::invalid_char('@', 4); + match err { + ToonError::InvalidCharacter { char, position } => { + assert_eq!(char, '@'); + assert_eq!(position, 4); + } + _ => panic!("Expected InvalidCharacter error"), + } + + let err = ToonError::type_mismatch("string", "number"); + match err { + ToonError::TypeMismatch { expected, found } => { + assert_eq!(expected, "string"); + assert_eq!(found, "number"); + } + _ => panic!("Expected TypeMismatch error"), + } + + let ctx = ErrorContext::new("bad input"); + let err = ToonError::parse_error(1, 2, "oops").with_context(ctx.clone()); + match err { + ToonError::ParseError { + context: Some(context), + .. + } => { + assert_eq!(context, Box::new(ctx.clone())); + } + _ => panic!("Expected ParseError with context"), + } + + let err = ToonError::length_mismatch(2, 1).with_context(ctx.clone()); + match err { + ToonError::LengthMismatch { + context: Some(context), + .. + } => { + assert_eq!(context, Box::new(ctx)); + } + _ => panic!("Expected LengthMismatch with context"), + } + + let err = ToonError::InvalidInput("nope".to_string()); + let untouched = err.clone().with_context(ErrorContext::new("unused")); + assert_eq!(err, untouched); + + let err = ToonError::parse_error(1, 1, "bad").with_suggestion("fix it"); + match err { + ToonError::ParseError { + context: Some(context), + .. + } => { + assert_eq!(context.suggestion.as_deref(), Some("fix it")); + } + _ => panic!("Expected ParseError with suggestion"), + } +} diff --git a/tests/options.rs b/tests/options.rs new file mode 100644 index 0000000..a44d8da --- /dev/null +++ b/tests/options.rs @@ -0,0 +1,43 @@ +use toon_format::types::{DecodeOptions, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}; +use toon_format::Delimiter; + +#[test] +fn test_indent_helpers() { + let indent = Indent::Spaces(2); + assert_eq!(indent.get_string(0), ""); + assert_eq!(indent.get_string(3).len(), 6); + assert_eq!(indent.get_spaces(), 2); + + let indent = Indent::Spaces(0); + assert_eq!(indent.get_string(2), ""); +} + +#[test] +fn test_encode_options_setters() { + let opts = EncodeOptions::new() + .with_delimiter(Delimiter::Pipe) + .with_key_folding(KeyFoldingMode::Safe) + .with_flatten_depth(2) + .with_spaces(4); + + assert_eq!(opts.delimiter, Delimiter::Pipe); + assert_eq!(opts.key_folding, KeyFoldingMode::Safe); + assert_eq!(opts.flatten_depth, 2); + assert_eq!(opts.indent, Indent::Spaces(4)); +} + +#[test] +fn test_decode_options_setters() { + let opts = DecodeOptions::new() + .with_strict(false) + .with_delimiter(Delimiter::Pipe) + .with_coerce_types(false) + .with_indent(Indent::Spaces(4)) + .with_expand_paths(PathExpansionMode::Safe); + + assert!(!opts.strict); + assert_eq!(opts.delimiter, Some(Delimiter::Pipe)); + assert!(!opts.coerce_types); + assert_eq!(opts.indent, Indent::Spaces(4)); + assert_eq!(opts.expand_paths, PathExpansionMode::Safe); +} diff --git a/tests/value.rs b/tests/value.rs new file mode 100644 index 0000000..872fd0d --- /dev/null +++ b/tests/value.rs @@ -0,0 +1,161 @@ +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use indexmap::IndexMap; +use serde_json::json; +use toon_format::types::{IntoJsonValue, JsonValue, Number}; + +#[test] +fn test_number_from_f64_rejects_non_finite() { + assert!(Number::from_f64(f64::NAN).is_none()); + assert!(Number::from_f64(f64::INFINITY).is_none()); + assert!(Number::from_f64(f64::NEG_INFINITY).is_none()); + assert!(Number::from_f64(1.5).is_some()); +} + +#[test] +fn test_number_integer_checks() { + let float_int = Number::Float(42.0); + assert!(float_int.is_i64()); + assert!(float_int.is_u64()); + + let float_frac = Number::Float(42.5); + assert!(!float_frac.is_i64()); + assert!(!float_frac.is_u64()); + + let float_max = Number::Float(i64::MAX as f64); + assert!(!float_max.is_i64()); + assert_eq!(float_max.as_i64(), Some(i64::MAX)); + + let float_neg = Number::Float(-1.0); + assert!(!float_neg.is_u64()); +} + +#[test] +fn test_number_as_conversions() { + let too_large = Number::PosInt(i64::MAX as u64 + 1); + assert_eq!(too_large.as_i64(), None); + + let neg = Number::NegInt(-5); + assert_eq!(neg.as_u64(), None); + + let float_exact = Number::Float(7.0); + assert_eq!(float_exact.as_i64(), Some(7)); + assert_eq!(float_exact.as_u64(), Some(7)); + + let float_frac = Number::Float(7.25); + assert_eq!(float_frac.as_i64(), None); + assert_eq!(float_frac.as_u64(), None); + + let float_nan = Number::Float(f64::NAN); + assert!(!float_nan.is_integer()); +} + +#[test] +fn test_number_display_nan() { + let value = Number::from(f64::NAN); + assert_eq!(format!("{value}"), "0"); +} + +#[test] +fn test_json_value_accessors_and_take() { + let mut obj = IndexMap::new(); + obj.insert("a".to_string(), JsonValue::Number(Number::from(1))); + + let mut value = JsonValue::Object(obj); + assert!(value.is_object()); + assert_eq!(value.type_name(), "object"); + assert_eq!(value.get("a").and_then(JsonValue::as_i64), Some(1)); + + value + .as_object_mut() + .unwrap() + .insert("b".to_string(), JsonValue::String("hi".to_string())); + assert_eq!(value.get("b").and_then(JsonValue::as_str), Some("hi")); + + let mut arr = JsonValue::Array(vec![JsonValue::Bool(true)]); + assert!(arr.is_array()); + arr.as_array_mut().unwrap().push(JsonValue::Null); + assert_eq!(arr.as_array().unwrap().len(), 2); + + let mut taken = JsonValue::String("take".to_string()); + let prior = taken.take(); + assert!(matches!(taken, JsonValue::Null)); + assert_eq!(prior.as_str(), Some("take")); +} + +#[test] +fn test_json_value_indexing_success() { + let mut arr = JsonValue::Array(vec![JsonValue::Number(Number::from(1)), JsonValue::Null]); + assert_eq!(arr[0].as_i64(), Some(1)); + arr[1] = JsonValue::Bool(true); + assert_eq!(arr[1].as_bool(), Some(true)); + + let mut obj = IndexMap::new(); + obj.insert("key".to_string(), JsonValue::Bool(false)); + let mut value = JsonValue::Object(obj); + + assert_eq!(value["key"].as_bool(), Some(false)); + value["key"] = JsonValue::Bool(true); + assert_eq!(value["key"].as_bool(), Some(true)); + + let owned_key = "key".to_string(); + assert_eq!(value[owned_key].as_bool(), Some(true)); +} + +#[test] +fn test_json_value_indexing_panics() { + let value = JsonValue::Null; + let err = catch_unwind(AssertUnwindSafe(|| { + let _ = &value["missing"]; + })); + assert!(err.is_err()); + + let empty_array = JsonValue::Array(Vec::new()); + let err = catch_unwind(AssertUnwindSafe(|| { + let _ = &empty_array[1]; + })); + assert!(err.is_err()); + + let mut not_array = JsonValue::Null; + let err = catch_unwind(AssertUnwindSafe(|| { + not_array[0] = JsonValue::Null; + })); + assert!(err.is_err()); + + let empty_object = JsonValue::Object(IndexMap::new()); + let err = catch_unwind(AssertUnwindSafe(|| { + let _ = &empty_object["absent"]; + })); + assert!(err.is_err()); +} + +#[test] +fn test_json_value_conversions() { + let json_value = json!({"a": [1, 2], "b": {"c": true}}); + let value = JsonValue::from(json_value.clone()); + let roundtrip: serde_json::Value = value.clone().into(); + assert_eq!(roundtrip, json_value); + + let nan_value = JsonValue::Number(Number::Float(f64::NAN)); + let json_nan: serde_json::Value = nan_value.into(); + assert_eq!(json_nan, json!(null)); +} + +#[test] +fn test_into_json_value_trait() { + let json_value = json!({"a": 1}); + let owned = json_value.into_json_value(); + assert_eq!(owned.get("a").and_then(JsonValue::as_i64), Some(1)); + + let json_value = json!({"b": true}); + let borrowed = (&json_value).into_json_value(); + assert_eq!(borrowed.get("b").and_then(JsonValue::as_bool), Some(true)); + + let value = JsonValue::Bool(false); + let cloned = value.into_json_value(); + assert!(matches!(cloned, JsonValue::Bool(false))); + + let value = JsonValue::Bool(true); + let borrowed = (&value).into_json_value(); + assert!(matches!(borrowed, JsonValue::Bool(true))); +} From 0ff027407aba5e4a7c5db78ce2ded4fd049d5784 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 18:59:52 +0000 Subject: [PATCH 20/24] test: cover parser and scanner errors --- tests/parser_errors.rs | 81 +++++++++++++++++++++++++++++++++++++++++ tests/scanner_errors.rs | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 tests/parser_errors.rs create mode 100644 tests/scanner_errors.rs diff --git a/tests/parser_errors.rs b/tests/parser_errors.rs new file mode 100644 index 0000000..1353d8b --- /dev/null +++ b/tests/parser_errors.rs @@ -0,0 +1,81 @@ +use serde_json::Value; +use toon_format::{decode, decode_strict, DecodeOptions, Delimiter}; + +#[test] +fn test_strict_rejects_multiple_root_values() { + let err = decode_strict::("hello\nworld").unwrap_err(); + assert!(err + .to_string() + .contains("Multiple values at root level are not allowed")); +} + +#[test] +fn test_strict_invalid_unquoted_key() { + let err = decode_strict::("bad-key: 1").unwrap_err(); + assert!(err.to_string().contains("Invalid unquoted key")); +} + +#[test] +fn test_strict_missing_colon_in_object() { + let err = decode_strict::("obj:\n key").unwrap_err(); + assert!(err + .to_string() + .contains("Expected ':' after 'key' in object context")); +} + +#[test] +fn test_array_header_hash_marker_rejected() { + let err = decode_strict::("items[#2]: a,b").unwrap_err(); + assert!(err + .to_string() + .contains("Length marker '#' is not supported")); +} + +#[test] +fn test_array_header_missing_right_bracket() { + let err = decode_strict::("items[1|: a|b").unwrap_err(); + assert!(err.to_string().contains("Expected ']'")); +} + +#[test] +fn test_tabular_header_requires_newline() { + let err = decode_strict::("items[1]{a}: 1").unwrap_err(); + assert!(err + .to_string() + .contains("Expected newline after tabular array header")); +} + +#[test] +fn test_tabular_row_missing_delimiter() { + let err = decode_strict::("items[1]{a,b}:\n 1 2").unwrap_err(); + assert!(err.to_string().contains("Expected delimiter")); +} + +#[test] +fn test_tabular_blank_line_strict() { + let err = decode_strict::("items[2]{a}:\n 1\n\n 2").unwrap_err(); + assert!(err + .to_string() + .contains("Blank lines are not allowed inside tabular arrays")); +} + +#[test] +fn test_inline_array_missing_delimiter_strict() { + let err = decode_strict::("items[2]: a b").unwrap_err(); + assert!(err.to_string().contains("Expected delimiter")); +} + +#[test] +fn test_list_array_blank_line_strict() { + let err = decode_strict::("items[2]:\n - a\n\n - b").unwrap_err(); + assert!(err + .to_string() + .contains("Blank lines are not allowed inside list arrays")); +} + +#[test] +fn test_array_header_delimiter_mismatch() { + let opts = DecodeOptions::new().with_delimiter(Delimiter::Pipe); + let err = decode::("items[2,]: a,b", &opts).unwrap_err(); + assert!(err.to_string().contains("Detected delimiter")); +} diff --git a/tests/scanner_errors.rs b/tests/scanner_errors.rs new file mode 100644 index 0000000..a4d727e --- /dev/null +++ b/tests/scanner_errors.rs @@ -0,0 +1,68 @@ +use toon_format::decode::scanner::{Scanner, Token}; +use toon_format::ToonError; + +#[test] +fn test_tabs_in_indentation_rejected() { + let mut scanner = Scanner::new("\tkey: value"); + let err = scanner.scan_token().unwrap_err(); + assert!(err + .to_string() + .contains("Tabs are not allowed in indentation")); +} + +#[test] +fn test_scan_quoted_string_invalid_escape() { + let mut scanner = Scanner::new(r#""bad\x""#); + let err = scanner.scan_token().unwrap_err(); + assert!(err.to_string().contains("Invalid escape sequence")); +} + +#[test] +fn test_scan_quoted_string_unterminated() { + let mut scanner = Scanner::new("\"unterminated"); + let err = scanner.scan_token().unwrap_err(); + assert!(matches!(err, ToonError::UnexpectedEof)); +} + +#[test] +fn test_parse_value_string_invalid_escape() { + let scanner = Scanner::new(""); + let err = scanner.parse_value_string(r#""bad\x""#).unwrap_err(); + assert!(err.to_string().contains("Invalid escape sequence")); +} + +#[test] +fn test_parse_value_string_unexpected_trailing_chars() { + let scanner = Scanner::new(""); + let err = scanner + .parse_value_string(r#""hello" trailing"#) + .unwrap_err(); + assert!(err + .to_string() + .contains("Unexpected characters after closing quote")); +} + +#[test] +fn test_parse_value_string_unterminated() { + let scanner = Scanner::new(""); + let err = scanner.parse_value_string(r#""missing"#).unwrap_err(); + assert!(err.to_string().contains("Unterminated string")); +} + +#[test] +fn test_scan_number_leading_zero_string() { + let mut scanner = Scanner::new("05"); + assert_eq!( + scanner.scan_token().unwrap(), + Token::String("05".to_string(), false) + ); +} + +#[test] +fn test_scan_number_trailing_char_string() { + let mut scanner = Scanner::new("1x"); + assert_eq!( + scanner.scan_token().unwrap(), + Token::String("1".to_string(), false) + ); +} From c94aca74dd625554ef0a8dba584d342954d6d532 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 19:07:18 +0000 Subject: [PATCH 21/24] test: fix clippy warnings in encode helpers --- tests/encode_helpers.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/encode_helpers.rs b/tests/encode_helpers.rs index 6d3625c..a7d44fb 100644 --- a/tests/encode_helpers.rs +++ b/tests/encode_helpers.rs @@ -1,7 +1,7 @@ use indexmap::IndexMap; use serde_json::json; -use toon_format::{encode_array, encode_object, EncodeOptions, ToonError}; use toon_format::types::{JsonValue, Number}; +use toon_format::{encode_array, encode_object, EncodeOptions, ToonError}; #[test] fn test_encode_array_and_object_with_json() { @@ -16,7 +16,7 @@ fn test_encode_array_and_object_with_json() { #[test] fn test_encode_array_object_type_mismatch() { - let err = encode_array(&json!({"a": 1}), &EncodeOptions::default()).unwrap_err(); + let err = encode_array(json!({"a": 1}), &EncodeOptions::default()).unwrap_err(); match err { ToonError::TypeMismatch { expected, found } => { assert_eq!(expected, "array"); @@ -25,7 +25,7 @@ fn test_encode_array_object_type_mismatch() { _ => panic!("Expected TypeMismatch for encode_array"), } - let err = encode_object(&json!(["a", "b"]), &EncodeOptions::default()).unwrap_err(); + let err = encode_object(json!(["a", "b"]), &EncodeOptions::default()).unwrap_err(); match err { ToonError::TypeMismatch { expected, found } => { assert_eq!(expected, "object"); From 9963e6297b59954608d23d87563590892225eca5 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 19:22:59 +0000 Subject: [PATCH 22/24] feat(serde): add serde-style api --- README.md | 32 ++++++++ src/lib.rs | 6 ++ src/serde.rs | 191 +++++++++++++++++++++++++++++++++++++++++++++ tests/serde_api.rs | 68 ++++++++++++++++ 4 files changed, 297 insertions(+) create mode 100644 src/serde.rs create mode 100644 tests/serde_api.rs diff --git a/README.md b/README.md index a86797e..68fc725 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,38 @@ fn main() -> Result<(), toon_format::ToonError> { Ok(()) } ``` + +### Serde-Style API + +Prefer serde_json-like helpers? Use `to_string`/`from_str` and friends: + +```rust +use serde::{Deserialize, Serialize}; +use toon_format::{from_reader, from_str, to_string, to_writer}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct User { + name: String, + age: u32, +} + +let user = User { + name: "Ada".to_string(), + age: 37, +}; + +let toon = to_string(&user)?; +let round_trip: User = from_str(&toon)?; + +let mut buffer = Vec::new(); +to_writer(&mut buffer, &user)?; +let round_trip: User = from_reader(buffer.as_slice())?; +# Ok::<(), toon_format::ToonError>(()) +``` + +Option-aware variants: `to_string_with_options`, `to_writer_with_options`, +`from_str_with_options`, `from_slice_with_options`, `from_reader_with_options`. + --- ## API Reference diff --git a/src/lib.rs b/src/lib.rs index 767488c..ac2778f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ pub mod constants; pub mod decode; pub mod encode; +pub mod serde; #[cfg(feature = "tui")] pub mod tui; pub mod types; @@ -38,6 +39,11 @@ pub use decode::{ decode_strict_with_options, }; pub use encode::{encode, encode_array, encode_default, encode_object}; +pub use serde::{ + from_reader, from_reader_with_options, from_slice, from_slice_with_options, from_str, + from_str_with_options, to_string, to_string_with_options, to_vec, to_writer, + to_writer_with_options, +}; pub use types::{DecodeOptions, Delimiter, EncodeOptions, Indent, ToonError}; pub use utils::{ literal::{is_keyword, is_literal_like}, diff --git a/src/serde.rs b/src/serde.rs new file mode 100644 index 0000000..eb25db1 --- /dev/null +++ b/src/serde.rs @@ -0,0 +1,191 @@ +use std::io::{Read, Write}; + +use ::serde::{de::DeserializeOwned, Serialize}; + +use crate::types::ToonResult; +use crate::{decode, encode, DecodeOptions, EncodeOptions, ToonError}; + +/// Serialize a value to a TOON string using default options. +/// +/// # Examples +/// ``` +/// use toon_format::to_string; +/// let toon = to_string(&serde_json::json!({"a": 1}))?; +/// assert!(toon.contains("a: 1")); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn to_string(value: &T) -> ToonResult { + encode(value, &EncodeOptions::default()) +} + +/// Serialize a value to a TOON string using custom options. +/// +/// # Examples +/// ``` +/// use toon_format::{to_string_with_options, Delimiter, EncodeOptions}; +/// let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe); +/// let toon = to_string_with_options(&serde_json::json!({"items": ["a", "b"]}), &opts)?; +/// assert!(toon.contains('|')); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn to_string_with_options(value: &T, opts: &EncodeOptions) -> ToonResult { + encode(value, opts) +} + +/// Serialize a value to a UTF-8 byte vector using default options. +/// +/// # Examples +/// ``` +/// use toon_format::to_vec; +/// let bytes = to_vec(&serde_json::json!({"a": 1}))?; +/// assert!(!bytes.is_empty()); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn to_vec(value: &T) -> ToonResult> { + Ok(to_string(value)?.into_bytes()) +} + +/// Serialize a value to a writer using default options. +/// +/// # Examples +/// ``` +/// use toon_format::to_writer; +/// let mut buffer = Vec::new(); +/// to_writer(&mut buffer, &serde_json::json!({"a": 1}))?; +/// assert!(!buffer.is_empty()); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn to_writer(mut writer: W, value: &T) -> ToonResult<()> { + let encoded = to_string(value)?; + writer + .write_all(encoded.as_bytes()) + .map_err(|err| ToonError::InvalidInput(format!("Failed to write output: {err}"))) +} + +/// Serialize a value to a writer using custom options. +/// +/// # Examples +/// ``` +/// use toon_format::{to_writer_with_options, Delimiter, EncodeOptions}; +/// let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe); +/// let mut buffer = Vec::new(); +/// to_writer_with_options(&mut buffer, &serde_json::json!({"items": ["a", "b"]}), &opts)?; +/// assert!(std::str::from_utf8(&buffer).expect("valid UTF-8").contains('|')); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn to_writer_with_options( + mut writer: W, + value: &T, + opts: &EncodeOptions, +) -> ToonResult<()> { + let encoded = to_string_with_options(value, opts)?; + writer + .write_all(encoded.as_bytes()) + .map_err(|err| ToonError::InvalidInput(format!("Failed to write output: {err}"))) +} + +/// Deserialize a value from a TOON string using default options. +/// +/// # Examples +/// ``` +/// use toon_format::from_str; +/// let value: serde_json::Value = from_str("a: 1")?; +/// assert_eq!(value, serde_json::json!({"a": 1})); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn from_str(input: &str) -> ToonResult { + decode(input, &DecodeOptions::default()) +} + +/// Deserialize a value from a TOON string using custom options. +/// +/// # Examples +/// ``` +/// use toon_format::{from_str_with_options, DecodeOptions}; +/// let opts = DecodeOptions::new().with_strict(false); +/// let value: serde_json::Value = from_str_with_options("items[2]: a", &opts)?; +/// assert_eq!(value, serde_json::json!({"items": ["a"]})); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn from_str_with_options( + input: &str, + opts: &DecodeOptions, +) -> ToonResult { + decode(input, opts) +} + +/// Deserialize a value from UTF-8 bytes using default options. +/// +/// # Examples +/// ``` +/// use toon_format::from_slice; +/// let value: serde_json::Value = from_slice(b"a: 1")?; +/// assert_eq!(value, serde_json::json!({"a": 1})); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn from_slice(input: &[u8]) -> ToonResult { + let s = std::str::from_utf8(input) + .map_err(|err| ToonError::InvalidInput(format!("Input is not valid UTF-8: {err}")))?; + from_str(s) +} + +/// Deserialize a value from UTF-8 bytes using custom options. +/// +/// # Examples +/// ``` +/// use toon_format::{from_slice_with_options, DecodeOptions}; +/// let opts = DecodeOptions::new().with_strict(false); +/// let value: serde_json::Value = from_slice_with_options(b"items[2]: a", &opts)?; +/// assert_eq!(value, serde_json::json!({"items": ["a"]})); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn from_slice_with_options( + input: &[u8], + opts: &DecodeOptions, +) -> ToonResult { + let s = std::str::from_utf8(input) + .map_err(|err| ToonError::InvalidInput(format!("Input is not valid UTF-8: {err}")))?; + from_str_with_options(s, opts) +} + +/// Deserialize a value from a reader using default options. +/// +/// # Examples +/// ``` +/// use std::io::Cursor; +/// use toon_format::from_reader; +/// let mut reader = Cursor::new("a: 1"); +/// let value: serde_json::Value = from_reader(&mut reader)?; +/// assert_eq!(value, serde_json::json!({"a": 1})); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn from_reader(mut reader: R) -> ToonResult { + let mut buf = Vec::new(); + reader + .read_to_end(&mut buf) + .map_err(|err| ToonError::InvalidInput(format!("Failed to read input: {err}")))?; + from_slice(&buf) +} + +/// Deserialize a value from a reader using custom options. +/// +/// # Examples +/// ``` +/// use std::io::Cursor; +/// use toon_format::{from_reader_with_options, DecodeOptions}; +/// let opts = DecodeOptions::new().with_strict(false); +/// let mut reader = Cursor::new("items[2]: a"); +/// let value: serde_json::Value = from_reader_with_options(&mut reader, &opts)?; +/// assert_eq!(value, serde_json::json!({"items": ["a"]})); +/// # Ok::<(), toon_format::ToonError>(()) +/// ``` +pub fn from_reader_with_options( + mut reader: R, + opts: &DecodeOptions, +) -> ToonResult { + let mut buf = Vec::new(); + reader + .read_to_end(&mut buf) + .map_err(|err| ToonError::InvalidInput(format!("Failed to read input: {err}")))?; + from_slice_with_options(&buf, opts) +} diff --git a/tests/serde_api.rs b/tests/serde_api.rs new file mode 100644 index 0000000..a754cb5 --- /dev/null +++ b/tests/serde_api.rs @@ -0,0 +1,68 @@ +use std::io::Cursor; + +use serde::{Deserialize, Serialize}; +use serde_json::json; +use toon_format::{ + from_reader, from_slice, from_str, from_str_with_options, to_string, to_string_with_options, + to_vec, to_writer, DecodeOptions, Delimiter, EncodeOptions, ToonError, +}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct User { + name: String, + age: u32, +} + +#[test] +fn test_round_trip_string_api() { + let user = User { + name: "Ada".to_string(), + age: 37, + }; + let encoded = to_string(&user).unwrap(); + let decoded: User = from_str(&encoded).unwrap(); + assert_eq!(decoded, user); +} + +#[test] +fn test_writer_reader_round_trip() { + let user = User { + name: "Turing".to_string(), + age: 41, + }; + let mut buffer = Vec::new(); + to_writer(&mut buffer, &user).unwrap(); + + let mut reader = Cursor::new(buffer); + let decoded: User = from_reader(&mut reader).unwrap(); + assert_eq!(decoded, user); +} + +#[test] +fn test_options_wiring() { + let data = json!({"items": ["a", "b"]}); + let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe); + let encoded = to_string_with_options(&data, &opts).unwrap(); + assert!(encoded.contains('|')); + + let decode_opts = DecodeOptions::new().with_strict(false); + let decoded: serde_json::Value = from_str_with_options("items[2]: a", &decode_opts).unwrap(); + assert_eq!(decoded, json!({"items": ["a"]})); +} + +#[test] +fn test_vec_and_slice_api() { + let user = User { + name: "Grace".to_string(), + age: 60, + }; + let bytes = to_vec(&user).unwrap(); + let decoded: User = from_slice(&bytes).unwrap(); + assert_eq!(decoded, user); +} + +#[test] +fn test_from_slice_invalid_utf8() { + let err = from_slice::(&[0xff]).unwrap_err(); + assert!(matches!(err, ToonError::InvalidInput(_))); +} From d23474f2c8e993b2657c84ecb08c397aa37e7709 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 20:07:20 +0000 Subject: [PATCH 23/24] docs: refresh documentation and examples --- README.md | 61 +++-- docs/CLI.md | 65 ++++++ examples/arrays.rs | 6 + examples/arrays_of_arrays.rs | 6 + examples/decode_strict.rs | 6 + examples/delimiters.rs | 6 + examples/empty_and_root.rs | 6 + examples/mixed_arrays.rs | 6 + examples/objects.rs | 6 + examples/round_trip.rs | 6 + examples/structs.rs | 6 + examples/tabular.rs | 6 + src/constants.rs | 18 ++ src/decode/expansion.rs | 47 ++++ src/decode/parser.rs | 29 +++ src/decode/scanner.rs | 208 +++++++++++++++++ src/decode/validation.rs | 52 ++++- src/encode/folding.rs | 35 +++ src/encode/primitives.rs | 29 +++ src/encode/writer.rs | 196 +++++++++++++++- src/lib.rs | 30 ++- src/tui/app.rs | 30 +++ src/tui/components/diff_viewer.rs | 30 +++ src/tui/components/editor.rs | 30 +++ src/tui/components/file_browser.rs | 71 ++++++ src/tui/components/help_screen.rs | 28 +++ src/tui/components/history_panel.rs | 30 +++ src/tui/components/repl_panel.rs | 28 +++ src/tui/components/settings_panel.rs | 30 +++ src/tui/components/stats_bar.rs | 30 +++ src/tui/components/status_bar.rs | 30 +++ src/tui/events.rs | 24 ++ src/tui/keybindings.rs | 35 +++ src/tui/mod.rs | 7 + src/tui/repl_command.rs | 36 ++- src/tui/state/app_state.rs | 263 +++++++++++++++++++++ src/tui/state/editor_state.rs | 107 +++++++++ src/tui/state/file_state.rs | 197 ++++++++++++++++ src/tui/state/repl_state.rs | 152 ++++++++++++- src/tui/theme.rs | 213 +++++++++++++++++ src/tui/ui.rs | 14 ++ src/types/delimiter.rs | 40 +++- src/types/errors.rs | 160 +++++++++++++ src/types/folding.rs | 29 ++- src/types/options.rs | 145 ++++++++++++ src/types/value.rs | 328 ++++++++++++++++++++++++++- src/utils/literal.rs | 39 +++- src/utils/mod.rs | 18 ++ src/utils/number.rs | 20 ++ src/utils/string.rs | 59 ++++- src/utils/validation.rs | 25 ++ 51 files changed, 3031 insertions(+), 47 deletions(-) create mode 100644 docs/CLI.md create mode 100644 examples/arrays.rs create mode 100644 examples/arrays_of_arrays.rs create mode 100644 examples/decode_strict.rs create mode 100644 examples/delimiters.rs create mode 100644 examples/empty_and_root.rs create mode 100644 examples/mixed_arrays.rs create mode 100644 examples/objects.rs create mode 100644 examples/round_trip.rs create mode 100644 examples/structs.rs create mode 100644 examples/tabular.rs diff --git a/README.md b/README.md index 68fc725..92038df 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ [![Crates.io](https://img.shields.io/crates/v/toon-format.svg)](https://crates.io/crates/toon-format) [![Documentation](https://docs.rs/toon-format/badge.svg)](https://docs.rs/toon-format) -[![Spec v2.0](https://img.shields.io/badge/spec-v2.0-brightgreen.svg)](https://github.com/toon-format/spec/blob/main/SPEC.md) +[![Spec v3.0](https://img.shields.io/badge/spec-v3.0-brightgreen.svg)](https://github.com/toon-format/spec/blob/main/SPEC.md) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) [![Tests](https://img.shields.io/badge/tests-%20passing-success.svg)]() **Token-Oriented Object Notation (TOON)** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. -This crate provides the official, **spec-compliant Rust implementation** of TOON v2.0 with v1.5 optional features, offering both a library (`toon-format`) and a full-featured command-line tool (`toon`). +This crate provides the official, **spec-compliant Rust implementation** of TOON v3.0 with v1.5 optional features, offering both a library (`toon-format`) and a full-featured command-line tool (`toon`). ## Quick Example @@ -32,7 +32,7 @@ users[2]{id,name}: ## Features - **Generic API**: Works with any `Serialize`/`Deserialize` type - custom structs, enums, JSON values, and more -- **Spec-Compliant**: Fully compliant with [TOON Specification v2.0](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Spec-Compliant**: Fully compliant with [TOON Specification v3.0](https://github.com/toon-format/spec/blob/main/SPEC.md) - **v1.5 Optional Features**: Key folding and path expansion - **Safe & Performant**: Built with safe, fast Rust - **Powerful CLI**: Full-featured command-line tool @@ -484,6 +484,8 @@ See [docs/TUI.md](docs/TUI.md) for complete documentation and keyboard shortcuts ## CLI Usage +See [docs/CLI.md](docs/CLI.md) for the full option reference and behavior details. + ### Basic Commands ```bash @@ -495,6 +497,9 @@ toon data.toon # Decode toon -e data.txt # Force encode toon -d output.txt # Force decode +# Write to a file +toon input.json -o output.toon + # Pipe from stdin cat data.json | toon echo '{"name": "Alice"}' | toon -e @@ -514,7 +519,7 @@ toon data.json --indent 4 toon data.json --fold-keys toon data.json --fold-keys --flatten-depth 2 -# Show statistics +# Show statistics (requires cli-stats feature) toon data.json --stats ``` @@ -596,25 +601,49 @@ match decode_strict::("items[3]: a,b") { --- +## Numeric Precision + +This implementation handles numbers as follows: + +- **Integers**: Values within `i64` range (`-9,223,372,036,854,775,808` to + `9,223,372,036,854,775,807`) are preserved exactly +- **Floating-point**: Numbers outside `i64` range or with decimal points use `f64`, + which provides ~15-17 significant digits of precision +- **Safe integers**: For JavaScript interoperability, integers up to `2^53 - 1` + (9,007,199,254,740,991) are safe + +### Special Values + +| Input | TOON Output | +|-------|-------------| +| `NaN` | `null` | +| `Infinity` | `null` | +| `-Infinity` | `null` | + +These conversions follow the TOON specification requirement that all values must be +JSON-compatible. + +--- + ## Examples -Run with `cargo run --example examples` to see all examples: -- `structs.rs` - Custom struct serialization -- `tabular.rs` - Tabular array formatting -- `arrays.rs` - Various array formats -- `arrays_of_arrays.rs` - Nested arrays -- `objects.rs` - Object encoding -- `mixed_arrays.rs` - Mixed-type arrays -- `delimiters.rs` - Custom delimiters -- `round_trip.rs` - Encode/decode round-trips -- `decode_strict.rs` - Strict validation -- `empty_and_root.rs` - Edge cases +Run a specific example with `cargo run --example `: +- `structs` - Custom struct serialization +- `tabular` - Tabular array formatting +- `arrays` - Various array formats +- `arrays_of_arrays` - Nested arrays +- `objects` - Object encoding +- `mixed_arrays` - Mixed-type arrays +- `delimiters` - Custom delimiters +- `round_trip` - Encode/decode round-trips +- `decode_strict` - Strict validation +- `empty_and_root` - Edge cases --- ## Resources -- 📖 [TOON Specification v2.0](https://github.com/toon-format/spec/blob/main/SPEC.md) +- 📖 [TOON Specification v3.0](https://github.com/toon-format/spec/blob/main/SPEC.md) - 📦 [Crates.io Package](https://crates.io/crates/toon-format) - 📚 [API Documentation](https://docs.rs/toon-format) - 🔧 [Main Repository (JS/TS)](https://github.com/toon-format/toon) diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 0000000..f1463dc --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,65 @@ +# CLI Usage + +The `toon` binary encodes JSON to TOON or decodes TOON to JSON. + +## Input Selection and Modes + +- `--interactive` / `-i` launches the TUI (requires the `tui` feature). +- `--encode` / `-e` forces JSON -> TOON. +- `--decode` / `-d` forces TOON -> JSON. +- If no mode is provided and the input path ends with `.json`, encode. +- If no mode is provided and the input path ends with `.toon`, decode. +- If reading from stdin (no input path or `-`), encode by default. + +## Output Behavior + +- Use `--output` / `-o` to write to a file; otherwise output goes to stdout. +- Decode writes a trailing newline to stdout if one is missing. +- Empty decode input prints `{}` and returns success. + +## Flags + +- `-i`, `--interactive`: Launch interactive TUI mode. +- `-o`, `--output `: Output file path. +- `-e`, `--encode`: Force encode mode (JSON -> TOON). +- `-d`, `--decode`: Force decode mode (TOON -> JSON). +- `--stats`: Show token count and savings (encode only, requires `cli-stats` feature). +- `--delimiter `: Set array delimiter (encode only). +- `--indent `: Set TOON indentation spaces (encode only). +- `--fold-keys`: Enable key folding (encode only). +- `--flatten-depth `: Limit key folding depth (requires `--fold-keys`). +- `--json-indent `: Pretty-print JSON with N spaces (decode only). +- `--no-strict`: Disable strict validation (decode only). +- `--no-coerce`: Disable type coercion (decode only). +- `--expand-paths`: Enable path expansion for dotted keys (decode only). + +## Examples + +```bash +# Auto-detect from extension +toon data.json # Encode +toon data.toon # Decode + +# Force mode +toon -e data.txt # Force encode +toon -d data.txt # Force decode + +# Output file +toon input.json -o output.toon + +# Pipe from stdin +cat data.json | toon +echo '{"name": "Alice"}' | toon -e + +# Encode options +toon data.json --delimiter pipe +toon data.json --indent 4 +toon data.json --fold-keys --flatten-depth 2 +toon data.json --stats + +# Decode options +toon data.toon --json-indent 2 +toon data.toon --no-strict +toon data.toon --no-coerce +toon data.toon --expand-paths +``` diff --git a/examples/arrays.rs b/examples/arrays.rs new file mode 100644 index 0000000..b683fad --- /dev/null +++ b/examples/arrays.rs @@ -0,0 +1,6 @@ +#[path = "parts/arrays.rs"] +mod arrays; + +fn main() { + arrays::arrays(); +} diff --git a/examples/arrays_of_arrays.rs b/examples/arrays_of_arrays.rs new file mode 100644 index 0000000..17dd177 --- /dev/null +++ b/examples/arrays_of_arrays.rs @@ -0,0 +1,6 @@ +#[path = "parts/arrays_of_arrays.rs"] +mod arrays_of_arrays; + +fn main() { + arrays_of_arrays::arrays_of_arrays(); +} diff --git a/examples/decode_strict.rs b/examples/decode_strict.rs new file mode 100644 index 0000000..3bf204d --- /dev/null +++ b/examples/decode_strict.rs @@ -0,0 +1,6 @@ +#[path = "parts/decode_strict.rs"] +mod decode_strict; + +fn main() { + decode_strict::decode_strict(); +} diff --git a/examples/delimiters.rs b/examples/delimiters.rs new file mode 100644 index 0000000..3125648 --- /dev/null +++ b/examples/delimiters.rs @@ -0,0 +1,6 @@ +#[path = "parts/delimiters.rs"] +mod delimiters; + +fn main() { + delimiters::delimiters(); +} diff --git a/examples/empty_and_root.rs b/examples/empty_and_root.rs new file mode 100644 index 0000000..2c2d7e0 --- /dev/null +++ b/examples/empty_and_root.rs @@ -0,0 +1,6 @@ +#[path = "parts/empty_and_root.rs"] +mod empty_and_root; + +fn main() { + empty_and_root::empty_and_root(); +} diff --git a/examples/mixed_arrays.rs b/examples/mixed_arrays.rs new file mode 100644 index 0000000..ade739a --- /dev/null +++ b/examples/mixed_arrays.rs @@ -0,0 +1,6 @@ +#[path = "parts/mixed_arrays.rs"] +mod mixed_arrays; + +fn main() { + mixed_arrays::mixed_arrays(); +} diff --git a/examples/objects.rs b/examples/objects.rs new file mode 100644 index 0000000..c5f16dc --- /dev/null +++ b/examples/objects.rs @@ -0,0 +1,6 @@ +#[path = "parts/objects.rs"] +mod objects; + +fn main() { + objects::objects(); +} diff --git a/examples/round_trip.rs b/examples/round_trip.rs new file mode 100644 index 0000000..2778da1 --- /dev/null +++ b/examples/round_trip.rs @@ -0,0 +1,6 @@ +#[path = "parts/round_trip.rs"] +mod round_trip; + +fn main() { + round_trip::round_trip(); +} diff --git a/examples/structs.rs b/examples/structs.rs new file mode 100644 index 0000000..dfe21b2 --- /dev/null +++ b/examples/structs.rs @@ -0,0 +1,6 @@ +#[path = "parts/structs.rs"] +mod structs; + +fn main() { + structs::serde_structs(); +} diff --git a/examples/tabular.rs b/examples/tabular.rs new file mode 100644 index 0000000..1a0f390 --- /dev/null +++ b/examples/tabular.rs @@ -0,0 +1,6 @@ +#[path = "parts/tabular.rs"] +mod tabular; + +fn main() { + tabular::tabular(); +} diff --git a/src/constants.rs b/src/constants.rs index 9a57f2f..7cfd61a 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -23,11 +23,29 @@ pub const MAX_DEPTH: usize = 256; pub(crate) const QUOTED_KEY_MARKER: char = '\x00'; #[inline] +/// Returns true if the character has structural meaning in TOON. +/// +/// # Examples +/// ``` +/// use toon_format::constants::is_structural_char; +/// +/// assert!(is_structural_char('[')); +/// assert!(!is_structural_char('a')); +/// ``` pub fn is_structural_char(ch: char) -> bool { matches!(ch, '[' | ']' | '{' | '}' | ':' | '-') } #[inline] +/// Returns true if the string is a reserved TOON keyword. +/// +/// # Examples +/// ``` +/// use toon_format::constants::is_keyword; +/// +/// assert!(is_keyword("null")); +/// assert!(!is_keyword("hello")); +/// ``` pub fn is_keyword(s: &str) -> bool { KEYWORDS.contains(&s) } diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index 2162aa0..229b8b2 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -5,6 +5,15 @@ use crate::{ types::{is_identifier_segment, JsonValue as Value, PathExpansionMode, ToonError, ToonResult}, }; +/// Determine whether a dotted key should be expanded. +/// +/// # Examples +/// ``` +/// use toon_format::decode::expansion::should_expand_key; +/// use toon_format::types::PathExpansionMode; +/// +/// assert_eq!(should_expand_key("a.b", PathExpansionMode::Safe), Some(vec!["a", "b"])); +/// ``` pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option> { match mode { PathExpansionMode::Off => None, @@ -40,6 +49,18 @@ pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option } } +/// Merge a value into a nested object based on path segments. +/// +/// # Examples +/// ``` +/// use indexmap::IndexMap; +/// use serde_json::json; +/// use toon_format::decode::expansion::deep_merge_value; +/// use toon_format::types::JsonValue; +/// +/// let mut target = IndexMap::new(); +/// deep_merge_value(&mut target, &["a", "b"], JsonValue::from(json!(1)), true).unwrap(); +/// ``` pub fn deep_merge_value( target: &mut IndexMap, segments: &[&str], @@ -99,6 +120,20 @@ pub fn deep_merge_value( deep_merge_value(nested_obj, remaining_segments, value, strict) } +/// Expand dotted keys inside an object. +/// +/// # Examples +/// ``` +/// use indexmap::IndexMap; +/// use serde_json::json; +/// use toon_format::decode::expansion::expand_paths_in_object; +/// use toon_format::types::{JsonValue, PathExpansionMode}; +/// +/// let mut obj = IndexMap::new(); +/// obj.insert("a.b".to_string(), JsonValue::from(json!(1))); +/// let expanded = expand_paths_in_object(obj, PathExpansionMode::Safe, true).unwrap(); +/// assert!(expanded.contains_key("a")); +/// ``` pub fn expand_paths_in_object( obj: IndexMap, mode: PathExpansionMode, @@ -140,6 +175,18 @@ pub fn expand_paths_in_object( Ok(result) } +/// Recursively expand dotted keys within a JSON value. +/// +/// # Examples +/// ``` +/// use serde_json::json; +/// use toon_format::decode::expansion::expand_paths_recursive; +/// use toon_format::types::{JsonValue, PathExpansionMode}; +/// +/// let value = JsonValue::from(json!({"a.b": 1})); +/// let expanded = expand_paths_recursive(value, PathExpansionMode::Safe, true).unwrap(); +/// assert!(expanded.is_object()); +/// ``` pub fn expand_paths_recursive( value: Value, mode: PathExpansionMode, diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 80f23c4..76a1450 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -27,6 +27,16 @@ enum ArrayParseContext { } /// Parser that builds JSON values from a sequence of tokens. +/// +/// # Examples +/// ``` +/// use toon_format::decode::parser::Parser; +/// use toon_format::DecodeOptions; +/// +/// let mut parser = Parser::new("a: 1", DecodeOptions::default()).unwrap(); +/// let value = parser.parse().unwrap(); +/// assert!(value.is_object()); +/// ``` #[allow(unused)] pub struct Parser { scanner: Scanner, @@ -39,6 +49,15 @@ pub struct Parser { impl Parser { /// Create a new parser with the given input and options. + /// + /// # Examples + /// ``` + /// use toon_format::decode::parser::Parser; + /// use toon_format::DecodeOptions; + /// + /// let parser = Parser::new("a: 1", DecodeOptions::default()).unwrap(); + /// let _ = parser; + /// ``` pub fn new(input: &str, options: DecodeOptions) -> ToonResult { let input: Arc = Arc::from(input); let mut scanner = Scanner::from_shared_input(input.clone()); @@ -59,6 +78,16 @@ impl Parser { } /// Parse the input into a JSON value. + /// + /// # Examples + /// ``` + /// use toon_format::decode::parser::Parser; + /// use toon_format::DecodeOptions; + /// + /// let mut parser = Parser::new("a: 1", DecodeOptions::default()).unwrap(); + /// let value = parser.parse().unwrap(); + /// assert!(value.is_object()); + /// ``` pub fn parse(&mut self) -> ToonResult { if self.options.strict { self.validate_indentation(self.scanner.get_last_line_indent())?; diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 0c61996..d5ad39f 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -6,6 +6,14 @@ use crate::{ }; /// Tokens produced by the scanner during lexical analysis. +/// +/// # Examples +/// ``` +/// use toon_format::decode::scanner::Token; +/// +/// let token = Token::Colon; +/// let _ = token; +/// ``` #[derive(Debug, Clone, PartialEq)] pub enum Token { LeftBracket, @@ -25,6 +33,14 @@ pub enum Token { } /// Scanner that tokenizes TOON input into a sequence of tokens. +/// +/// # Examples +/// ``` +/// use toon_format::decode::scanner::Scanner; +/// +/// let scanner = Scanner::new("a: 1"); +/// let _ = scanner; +/// ``` pub struct Scanner { input: Arc, position: usize, @@ -47,10 +63,28 @@ struct CachedIndent { impl Scanner { /// Create a new scanner for the given input string. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("a: 1"); + /// let _ = scanner; + /// ``` pub fn new(input: &str) -> Self { Self::from_shared_input(Arc::from(input)) } + /// Create a scanner from a shared input buffer. + /// + /// # Examples + /// ``` + /// use std::sync::Arc; + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::from_shared_input(Arc::from("a: 1")); + /// let _ = scanner; + /// ``` pub fn from_shared_input(input: Arc) -> Self { Self { input, @@ -67,32 +101,94 @@ impl Scanner { } /// Set the active delimiter for tokenizing array elements. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// use toon_format::Delimiter; + /// + /// let mut scanner = Scanner::new("a: 1"); + /// scanner.set_active_delimiter(Some(Delimiter::Pipe)); + /// ``` pub fn set_active_delimiter(&mut self, delimiter: Option) { self.active_delimiter = delimiter; } + /// Enable or disable type coercion when scanning values. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new("a: 1"); + /// scanner.set_coerce_types(false); + /// ``` pub fn set_coerce_types(&mut self, coerce_types: bool) { self.coerce_types = coerce_types; } + /// Configure indentation handling based on strictness and width. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new("a: 1"); + /// scanner.configure_indentation(true, 2); + /// ``` pub fn configure_indentation(&mut self, strict: bool, indent_width: usize) { self.allow_tab_indent = !strict; self.indent_width = indent_width.max(1); } /// Get the current position (line, column). + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("a: 1"); + /// assert_eq!(scanner.current_position(), (1, 1)); + /// ``` pub fn current_position(&self) -> (usize, usize) { (self.line, self.column) } + /// Return the current line number. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("a: 1"); + /// assert_eq!(scanner.get_line(), 1); + /// ``` pub fn get_line(&self) -> usize { self.line } + /// Return the current column number. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("a: 1"); + /// assert_eq!(scanner.get_column(), 1); + /// ``` pub fn get_column(&self) -> usize { self.column } + /// Peek at the next character without advancing. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("a: 1"); + /// assert_eq!(scanner.peek(), Some('a')); + /// ``` pub fn peek(&self) -> Option { let bytes = self.input.as_bytes(); match bytes.get(self.position) { @@ -102,10 +198,28 @@ impl Scanner { } } + /// Count leading spaces from the current position. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new(" a: 1"); + /// assert_eq!(scanner.count_leading_spaces(), 2); + /// ``` pub fn count_leading_spaces(&mut self) -> usize { self.peek_indent() } + /// Count spaces immediately following a newline. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("\n a: 1"); + /// assert_eq!(scanner.count_spaces_after_newline(), 2); + /// ``` pub fn count_spaces_after_newline(&self) -> usize { let mut idx = self.position; if self.input.as_bytes().get(idx) != Some(&b'\n') { @@ -115,6 +229,15 @@ impl Scanner { self.count_indent_from(idx) } + /// Peek ahead by an offset without advancing. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let scanner = Scanner::new("abc"); + /// assert_eq!(scanner.peek_ahead(2), Some('c')); + /// ``` pub fn peek_ahead(&self, offset: usize) -> Option { let bytes = self.input.as_bytes(); let mut idx = self.position; @@ -136,6 +259,16 @@ impl Scanner { None } + /// Advance one character and return it. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new("ab"); + /// assert_eq!(scanner.advance(), Some('a')); + /// assert_eq!(scanner.advance(), Some('b')); + /// ``` pub fn advance(&mut self) -> Option { let bytes = self.input.as_bytes(); match bytes.get(self.position) { @@ -164,6 +297,16 @@ impl Scanner { } } + /// Skip contiguous space characters. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new(" a"); + /// scanner.skip_whitespace(); + /// assert_eq!(scanner.peek(), Some('a')); + /// ``` pub fn skip_whitespace(&mut self) { while let Some(ch) = self.peek() { if ch == ' ' { @@ -218,6 +361,14 @@ impl Scanner { } /// Scan the next token from the input. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::{Scanner, Token}; + /// + /// let mut scanner = Scanner::new("["); + /// assert_eq!(scanner.scan_token().unwrap(), Token::LeftBracket); + /// ``` pub fn scan_token(&mut self) -> ToonResult { if self.column == 1 { let mut indent_consumed = false; @@ -450,6 +601,16 @@ impl Scanner { false } + /// Return the indentation count for the last scanned line. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new(" a: 1"); + /// let _ = scanner.scan_token(); + /// let _ = scanner.get_last_line_indent(); + /// ``` pub fn get_last_line_indent(&self) -> usize { self.last_line_indent } @@ -530,6 +691,16 @@ impl Scanner { /// Read the rest of the current line (until newline or EOF). /// Returns the content and any leading spaces between the current token /// and the rest of the line. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new(" a: 1"); + /// let (content, leading) = scanner.read_rest_of_line_with_space_info(); + /// assert_eq!(leading, " "); + /// assert!(content.contains("a: 1")); + /// ``` pub fn read_rest_of_line_with_space_info(&mut self) -> (String, String) { let (content, leading_space) = self.read_rest_of_line_with_space_count(); let mut spaces = String::with_capacity(leading_space); @@ -540,6 +711,16 @@ impl Scanner { /// Read the rest of the current line (until newline or EOF). /// Returns the content and number of leading spaces between the current /// token and the rest of the line. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new(" a: 1"); + /// let (content, spaces) = scanner.read_rest_of_line_with_space_count(); + /// assert_eq!(spaces, 2); + /// assert!(content.contains("a: 1")); + /// ``` pub fn read_rest_of_line_with_space_count(&mut self) -> (String, usize) { let mut leading_space = 0usize; while matches!(self.peek(), Some(' ')) { @@ -562,11 +743,28 @@ impl Scanner { } /// Read the rest of the current line (until newline or EOF). + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// + /// let mut scanner = Scanner::new("a: 1"); + /// let content = scanner.read_rest_of_line(); + /// assert!(content.contains("a: 1")); + /// ``` pub fn read_rest_of_line(&mut self) -> String { self.read_rest_of_line_with_space_count().0 } /// Parse a complete value string into a token. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::{Scanner, Token}; + /// + /// let scanner = Scanner::new(""); + /// assert_eq!(scanner.parse_value_string("true").unwrap(), Token::Bool(true)); + /// ``` pub fn parse_value_string(&self, s: &str) -> ToonResult { let trimmed = s.trim(); @@ -668,6 +866,16 @@ impl Scanner { Ok(Token::String(trimmed.to_string(), false)) } + /// Detect the delimiter used in the input. + /// + /// # Examples + /// ``` + /// use toon_format::decode::scanner::Scanner; + /// use toon_format::Delimiter; + /// + /// let mut scanner = Scanner::new("a|b"); + /// assert_eq!(scanner.detect_delimiter(), Some(Delimiter::Pipe)); + /// ``` pub fn detect_delimiter(&mut self) -> Option { let saved_pos = self.position; diff --git a/src/decode/validation.rs b/src/decode/validation.rs index 29aab52..96d3b77 100644 --- a/src/decode/validation.rs +++ b/src/decode/validation.rs @@ -1,7 +1,15 @@ use crate::types::{ToonError, ToonResult}; use std::collections::HashSet; -/// Validate that array length matches expected value. +/// Validate that an array length matches the header value. +/// +/// # Examples +/// ``` +/// use toon_format::decode::validation::validate_array_length; +/// +/// assert!(validate_array_length(2, 2).is_ok()); +/// assert!(validate_array_length(2, 3).is_err()); +/// ``` pub fn validate_array_length(expected: usize, actual: usize) -> ToonResult<()> { if expected != actual { return Err(ToonError::length_mismatch(expected, actual)); @@ -9,7 +17,15 @@ pub fn validate_array_length(expected: usize, actual: usize) -> ToonResult<()> { Ok(()) } -/// Validate that array length is non-negative. +/// Validate that an array length is non-negative. +/// +/// # Examples +/// ``` +/// use toon_format::decode::validation::validate_array_length_non_negative; +/// +/// assert!(validate_array_length_non_negative(0).is_ok()); +/// assert!(validate_array_length_non_negative(-1).is_err()); +/// ``` pub fn validate_array_length_non_negative(length: i64) -> ToonResult<()> { if length < 0 { return Err(ToonError::InvalidInput( @@ -19,7 +35,15 @@ pub fn validate_array_length_non_negative(length: i64) -> ToonResult<()> { Ok(()) } -/// Validate field list for tabular arrays (no duplicates, non-empty names). +/// Validate that a tabular field list has unique entries. +/// +/// # Examples +/// ``` +/// use toon_format::decode::validation::validate_field_list; +/// +/// let fields = vec!["a".to_string(), "b".to_string()]; +/// assert!(validate_field_list(&fields).is_ok()); +/// ``` pub fn validate_field_list(fields: &[String]) -> ToonResult<()> { if fields.is_empty() { return Err(ToonError::InvalidInput( @@ -44,7 +68,15 @@ pub fn validate_field_list(fields: &[String]) -> ToonResult<()> { Ok(()) } -/// Validate that a tabular row has the expected number of values. +/// Validate that a tabular row length matches the header. +/// +/// # Examples +/// ``` +/// use toon_format::decode::validation::validate_row_length; +/// +/// assert!(validate_row_length(1, 2, 2).is_ok()); +/// assert!(validate_row_length(1, 2, 1).is_err()); +/// ``` pub fn validate_row_length( row_index: usize, expected_fields: usize, @@ -58,7 +90,17 @@ pub fn validate_row_length( Ok(()) } -/// Validate that detected and expected delimiters match. +/// Validate that the row delimiter matches the header. +/// +/// # Examples +/// ``` +/// use toon_format::decode::validation::validate_delimiter_consistency; +/// use toon_format::Delimiter; +/// +/// assert!( +/// validate_delimiter_consistency(Some(Delimiter::Comma), Some(Delimiter::Comma)).is_ok() +/// ); +/// ``` pub fn validate_delimiter_consistency( detected: Option, expected: Option, diff --git a/src/encode/folding.rs b/src/encode/folding.rs index 3ead315..323b21b 100644 --- a/src/encode/folding.rs +++ b/src/encode/folding.rs @@ -3,6 +3,19 @@ use std::collections::HashSet; use crate::types::{is_identifier_segment, JsonValue as Value, KeyFoldingMode}; /// Result of chain analysis for folding. +/// +/// # Examples +/// ``` +/// use std::collections::HashSet; +/// use serde_json::json; +/// use toon_format::encode::folding::analyze_foldable_chain; +/// use toon_format::types::JsonValue; +/// +/// let value: JsonValue = json!({"b": {"c": 1}}).into(); +/// let existing: HashSet<&str> = HashSet::new(); +/// let chain = analyze_foldable_chain("a", &value, usize::MAX, &existing).unwrap(); +/// assert_eq!(chain.folded_key, "a.b.c"); +/// ``` pub struct FoldableChain<'a> { /// The folded key path (e.g., "a.b.c") pub folded_key: String, @@ -23,6 +36,19 @@ fn is_single_key_object(value: &Value) -> Option<(&str, &Value)> { } /// Analyze if a key-value pair can be folded into dotted notation. +/// +/// # Examples +/// ``` +/// use std::collections::HashSet; +/// use serde_json::json; +/// use toon_format::encode::folding::analyze_foldable_chain; +/// use toon_format::types::JsonValue; +/// +/// let value: JsonValue = json!({"b": {"c": 1}}).into(); +/// let existing: HashSet<&str> = HashSet::new(); +/// let chain = analyze_foldable_chain("a", &value, usize::MAX, &existing).unwrap(); +/// assert_eq!(chain.depth_folded, 3); +/// ``` pub fn analyze_foldable_chain<'a>( key: &'a str, value: &'a Value, @@ -78,6 +104,15 @@ pub fn analyze_foldable_chain<'a>( }) } +/// Return true when folding should be applied for the current mode. +/// +/// # Examples +/// ``` +/// use toon_format::encode::folding::should_fold; +/// use toon_format::types::KeyFoldingMode; +/// +/// assert!(!should_fold(KeyFoldingMode::Off, &None)); +/// ``` pub fn should_fold(mode: KeyFoldingMode, chain: &Option) -> bool { match mode { KeyFoldingMode::Off => false, diff --git a/src/encode/primitives.rs b/src/encode/primitives.rs index a837782..2b8d0d9 100644 --- a/src/encode/primitives.rs +++ b/src/encode/primitives.rs @@ -1,3 +1,13 @@ +/// Returns true when a JSON value is a primitive. +/// +/// # Examples +/// ``` +/// use serde_json::json; +/// use toon_format::encode::primitives::is_primitive; +/// +/// assert!(is_primitive(&json!(42))); +/// assert!(!is_primitive(&json!([1, 2]))); +/// ``` pub fn is_primitive(value: &serde_json::Value) -> bool { matches!( value, @@ -8,11 +18,30 @@ pub fn is_primitive(value: &serde_json::Value) -> bool { ) } +/// Returns true when every JSON value in the slice is a primitive. +/// +/// # Examples +/// ``` +/// use serde_json::json; +/// use toon_format::encode::primitives::all_primitives; +/// +/// assert!(all_primitives(&[json!(1), json!(2)])); +/// assert!(!all_primitives(&[json!(1), json!({})])); +/// ``` pub fn all_primitives(values: &[serde_json::Value]) -> bool { values.iter().all(is_primitive) } /// Recursively normalize JSON values. +/// +/// # Examples +/// ``` +/// use serde_json::json; +/// use toon_format::encode::primitives::normalize_value; +/// +/// let value = json!({"a": [1, 2]}); +/// assert_eq!(normalize_value(value), json!({"a": [1, 2]})); +/// ``` pub fn normalize_value(value: serde_json::Value) -> serde_json::Value { match value { serde_json::Value::Null => serde_json::Value::Null, diff --git a/src/encode/writer.rs b/src/encode/writer.rs index ae11ec3..8334785 100644 --- a/src/encode/writer.rs +++ b/src/encode/writer.rs @@ -8,6 +8,16 @@ use crate::{ }; /// Writer that builds TOON output string from JSON values. +/// +/// # Examples +/// ``` +/// use toon_format::EncodeOptions; +/// use toon_format::encode::writer::Writer; +/// +/// let mut writer = Writer::new(EncodeOptions::default()); +/// writer.write_str("a: 1").unwrap(); +/// assert_eq!(writer.finish(), "a: 1"); +/// ``` pub struct Writer { buffer: String, pub(crate) options: EncodeOptions, @@ -18,6 +28,15 @@ pub struct Writer { impl Writer { /// Create a new writer with the given options. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let writer = Writer::new(EncodeOptions::default()); + /// let _ = writer; + /// ``` pub fn new(options: EncodeOptions) -> Self { let indent_unit = " ".repeat(options.indent.get_spaces()); Self { @@ -30,25 +49,80 @@ impl Writer { } /// Finish writing and return the complete TOON string. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_str("a: 1").unwrap(); + /// assert_eq!(writer.finish(), "a: 1"); + /// ``` pub fn finish(self) -> String { self.buffer } + /// Append a raw string to the output buffer. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_str("a").unwrap(); + /// assert_eq!(writer.finish(), "a"); + /// ``` pub fn write_str(&mut self, s: &str) -> ToonResult<()> { self.buffer.push_str(s); Ok(()) } + /// Append a character to the output buffer. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_char('x').unwrap(); + /// assert_eq!(writer.finish(), "x"); + /// ``` pub fn write_char(&mut self, ch: char) -> ToonResult<()> { self.buffer.push(ch); Ok(()) } + /// Append a newline to the output buffer. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_newline().unwrap(); + /// assert_eq!(writer.finish(), "\n"); + /// ``` pub fn write_newline(&mut self) -> ToonResult<()> { self.buffer.push('\n'); Ok(()) } + /// Write indentation for the requested depth. + /// + /// # Examples + /// ``` + /// use toon_format::{EncodeOptions, Indent}; + /// use toon_format::encode::writer::Writer; + /// + /// let opts = EncodeOptions::new().with_indent(Indent::Spaces(2)); + /// let mut writer = Writer::new(opts); + /// writer.write_indent(2).unwrap(); + /// assert_eq!(writer.finish(), " "); + /// ``` pub fn write_indent(&mut self, depth: usize) -> ToonResult<()> { if depth == 0 || self.indent_unit.is_empty() { return Ok(()); @@ -60,11 +134,34 @@ impl Writer { Ok(()) } + /// Write the active delimiter. + /// + /// # Examples + /// ``` + /// use toon_format::{Delimiter, EncodeOptions}; + /// use toon_format::encode::writer::Writer; + /// + /// let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe); + /// let mut writer = Writer::new(opts); + /// writer.write_delimiter().unwrap(); + /// assert_eq!(writer.finish(), "|"); + /// ``` pub fn write_delimiter(&mut self) -> ToonResult<()> { self.buffer.push(self.options.delimiter.as_char()); Ok(()) } + /// Write a key, quoting when needed. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_key("simple").unwrap(); + /// assert_eq!(writer.finish(), "simple"); + /// ``` pub fn write_key(&mut self, key: &str) -> ToonResult<()> { if is_valid_unquoted_key(key) { self.write_str(key) @@ -74,6 +171,16 @@ impl Writer { } /// Write an array header with key, length, and optional field list. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_array_header(Some("items"), 2, None, 0).unwrap(); + /// assert_eq!(writer.finish(), "items[2]:"); + /// ``` pub fn write_array_header( &mut self, key: Option<&str>, @@ -114,6 +221,16 @@ impl Writer { } /// Write an empty array header. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_empty_array_with_key(Some("items"), 0).unwrap(); + /// assert_eq!(writer.finish(), "items[0]:"); + /// ``` pub fn write_empty_array_with_key( &mut self, key: Option<&str>, @@ -136,6 +253,17 @@ impl Writer { self.write_char(':') } + /// Return true if a value needs quoting in the given context. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::utils::QuotingContext; + /// use toon_format::encode::writer::Writer; + /// + /// let writer = Writer::new(EncodeOptions::default()); + /// assert!(writer.needs_quoting("true", QuotingContext::ObjectValue)); + /// ``` pub fn needs_quoting(&self, s: &str, context: QuotingContext) -> bool { // Use active delimiter for array values, document delimiter for object values let delim_char = match context { @@ -145,6 +273,17 @@ impl Writer { needs_quoting(s, delim_char) } + /// Write a quoted and escaped string. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_quoted_string("a b").unwrap(); + /// assert_eq!(writer.finish(), "\"a b\""); + /// ``` pub fn write_quoted_string(&mut self, s: &str) -> ToonResult<()> { self.buffer.push('"'); escape_string_into(&mut self.buffer, s); @@ -152,6 +291,18 @@ impl Writer { Ok(()) } + /// Write a value, quoting only when needed. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::utils::QuotingContext; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_value("hello", QuotingContext::ObjectValue).unwrap(); + /// assert_eq!(writer.finish(), "hello"); + /// ``` pub fn write_value(&mut self, s: &str, context: QuotingContext) -> ToonResult<()> { if self.needs_quoting(s, context) { self.write_quoted_string(s) @@ -160,23 +311,64 @@ impl Writer { } } + /// Write a canonical number into the output buffer. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::types::Number; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_canonical_number(&Number::from(3.14f64)).unwrap(); + /// assert!(writer.finish().starts_with("3.14")); + /// ``` pub fn write_canonical_number(&mut self, n: &Number) -> ToonResult<()> { write_canonical_number_into(n, &mut self.buffer); Ok(()) } + /// Write a usize into the output buffer. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.write_usize(10).unwrap(); + /// assert_eq!(writer.finish(), "10"); + /// ``` pub fn write_usize(&mut self, value: usize) -> ToonResult<()> { let mut buf = itoa::Buffer::new(); self.buffer.push_str(buf.format(value as u64)); Ok(()) } - /// Push a new delimiter onto the stack (for nested arrays with different - /// delimiters). + /// Push a new delimiter onto the stack (for nested arrays with different delimiters). + /// + /// # Examples + /// ``` + /// use toon_format::{Delimiter, EncodeOptions}; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.push_active_delimiter(Delimiter::Pipe); + /// ``` pub fn push_active_delimiter(&mut self, delim: Delimiter) { self.active_delimiters.push(delim); } /// Pop the active delimiter, keeping at least one (the document default). + /// + /// # Examples + /// ``` + /// use toon_format::{Delimiter, EncodeOptions}; + /// use toon_format::encode::writer::Writer; + /// + /// let mut writer = Writer::new(EncodeOptions::default()); + /// writer.push_active_delimiter(Delimiter::Pipe); + /// writer.pop_active_delimiter(); + /// ``` pub fn pop_active_delimiter(&mut self) { if self.active_delimiters.len() > 1 { self.active_delimiters.pop(); diff --git a/src/lib.rs b/src/lib.rs index ac2778f..4cf7e26 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,25 +4,33 @@ //! designed for passing structured data to Large Language Models with //! significantly reduced token usage. //! -//! This crate reserves the `toon-format` namespace for the official Rust -//! implementation. Full implementation coming soon! +//! This crate provides the official, spec-compliant Rust implementation of +//! TOON v3.0, including library APIs plus the CLI/TUI (enabled by default). //! -//! ## Resources +//! ## Numeric Precision //! -//! - [TOON Specification](https://github.com/johannschopplich/toon/blob/main/SPEC.md) -//! - [Main Repository](https://github.com/johannschopplich/toon) -//! - [Other Implementations](https://github.com/johannschopplich/toon#other-implementations) +//! - Integers within `i64` range are preserved exactly +//! - Numbers outside `i64` range or with decimal points use `f64` (IEEE 754) +//! - `NaN`, `Infinity`, and `-Infinity` are converted to `null` //! -//! ## Example Usage (Future) +//! ## Example Usage //! -//! ```ignore -//! use toon_format::{encode, decode}; +//! ```rust +//! use serde_json::json; +//! use toon_format::{decode_default, encode_default}; //! //! let data = json!({"name": "Alice", "age": 30}); -//! let toon_string = encode(&data)?; -//! let decoded = decode(&toon_string)?; +//! let toon = encode_default(&data)?; +//! let decoded: serde_json::Value = decode_default(&toon)?; +//! assert_eq!(decoded, data); //! # Ok::<(), toon_format::ToonError>(()) //! ``` +//! +//! ## Resources +//! +//! - [TOON Specification](https://github.com/toon-format/spec/blob/main/SPEC.md) +//! - [Main Repository](https://github.com/toon-format/toon) +//! - [Other Implementations](https://github.com/toon-format/toon#other-implementations) #![warn(rustdoc::missing_crate_level_docs)] pub mod constants; diff --git a/src/tui/app.rs b/src/tui/app.rs index 9c74637..520462d 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -21,12 +21,29 @@ use crate::{ use crate::tui::state::ConversionStats; /// Main TUI application managing state, events, and rendering. +/// +/// # Examples +/// ```no_run +/// use toon_format::tui::app::TuiApp; +/// +/// let mut app = TuiApp::new(); +/// let _ = app; +/// ``` pub struct TuiApp<'a> { pub app_state: AppState<'a>, pub file_browser: FileBrowser, } impl<'a> TuiApp<'a> { + /// Create a new TUI application with default state. + /// + /// # Examples + /// ``` + /// use toon_format::tui::app::TuiApp; + /// + /// let app = TuiApp::new(); + /// let _ = app; + /// ``` pub fn new() -> Self { Self { app_state: AppState::new(), @@ -34,6 +51,19 @@ impl<'a> TuiApp<'a> { } } + /// Run the application loop on the provided terminal. + /// + /// # Examples + /// ```no_run + /// use std::io::stdout; + /// use ratatui::{backend::CrosstermBackend, Terminal}; + /// use toon_format::tui::app::TuiApp; + /// + /// let mut app = TuiApp::new(); + /// let backend = CrosstermBackend::new(stdout()); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// app.run(&mut terminal).unwrap(); + /// ``` pub fn run( &mut self, terminal: &mut ratatui::Terminal, diff --git a/src/tui/components/diff_viewer.rs b/src/tui/components/diff_viewer.rs index 9494fb8..9bc2668 100644 --- a/src/tui/components/diff_viewer.rs +++ b/src/tui/components/diff_viewer.rs @@ -9,9 +9,39 @@ use ratatui::{ use crate::tui::{state::AppState, theme::Theme}; +/// Diff viewer rendering for input/output comparison. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::DiffViewer, state::AppState, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let app = AppState::new(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| DiffViewer::render(f, f.area(), &app, &theme)) +/// .unwrap(); +/// ``` pub struct DiffViewer; impl DiffViewer { + /// Render the diff viewer. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::DiffViewer, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let app = AppState::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| DiffViewer::render(f, f.area(), &app, &theme)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, app: &AppState, theme: &Theme) { let block = Block::default() .borders(Borders::ALL) diff --git a/src/tui/components/editor.rs b/src/tui/components/editor.rs index 00fb043..8bd6c0b 100644 --- a/src/tui/components/editor.rs +++ b/src/tui/components/editor.rs @@ -8,9 +8,39 @@ use ratatui::{ use crate::tui::{state::AppState, theme::Theme}; +/// Editor panel rendering for input and output text. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::EditorComponent, state::AppState, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let mut app = AppState::new(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| EditorComponent::render(f, f.area(), f.area(), &mut app, &theme)) +/// .unwrap(); +/// ``` pub struct EditorComponent; impl EditorComponent { + /// Render the input and output editors. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::EditorComponent, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let mut app = AppState::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| EditorComponent::render(f, f.area(), f.area(), &mut app, &theme)) + /// .unwrap(); + /// ``` pub fn render( f: &mut Frame, input_area: Rect, diff --git a/src/tui/components/file_browser.rs b/src/tui/components/file_browser.rs index 4baf080..b818361 100644 --- a/src/tui/components/file_browser.rs +++ b/src/tui/components/file_browser.rs @@ -12,12 +12,29 @@ use ratatui::{ use crate::tui::{state::AppState, theme::Theme}; /// File browser state and rendering. +/// +/// # Examples +/// ``` +/// use toon_format::tui::components::FileBrowser; +/// +/// let browser = FileBrowser::new(); +/// let _ = browser; +/// ``` pub struct FileBrowser { pub selected_index: usize, pub scroll_offset: usize, } impl FileBrowser { + /// Create a new file browser instance. + /// + /// # Examples + /// ``` + /// use toon_format::tui::components::FileBrowser; + /// + /// let browser = FileBrowser::new(); + /// let _ = browser; + /// ``` pub fn new() -> Self { Self { selected_index: 0, @@ -25,6 +42,15 @@ impl FileBrowser { } } + /// Move the selection up by one row. + /// + /// # Examples + /// ``` + /// use toon_format::tui::components::FileBrowser; + /// + /// let mut browser = FileBrowser::new(); + /// browser.move_up(); + /// ``` pub fn move_up(&mut self) { if self.selected_index > 0 { self.selected_index -= 1; @@ -34,12 +60,31 @@ impl FileBrowser { } } + /// Move the selection down by one row. + /// + /// # Examples + /// ``` + /// use toon_format::tui::components::FileBrowser; + /// + /// let mut browser = FileBrowser::new(); + /// browser.move_down(10); + /// ``` pub fn move_down(&mut self, max: usize) { if self.selected_index < max.saturating_sub(1) { self.selected_index += 1; } } + /// Return the selected entry for a directory. + /// + /// # Examples + /// ``` + /// use std::path::Path; + /// use toon_format::tui::components::FileBrowser; + /// + /// let browser = FileBrowser::new(); + /// let _ = browser.get_selected_entry(Path::new(".")); + /// ``` pub fn get_selected_entry(&self, dir: &std::path::Path) -> Option { let entries = self.get_directory_entries(dir); if self.selected_index < entries.len() { @@ -54,10 +99,36 @@ impl FileBrowser { } } + /// Return the number of entries for a directory. + /// + /// # Examples + /// ``` + /// use std::path::Path; + /// use toon_format::tui::components::FileBrowser; + /// + /// let browser = FileBrowser::new(); + /// let _ = browser.get_entry_count(Path::new(".")); + /// ``` pub fn get_entry_count(&self, dir: &std::path::Path) -> usize { self.get_directory_entries(dir).len() } + /// Render the file browser panel. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::FileBrowser, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let mut app = AppState::new(); + /// let mut browser = FileBrowser::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| browser.render(f, f.area(), &app, &theme)) + /// .unwrap(); + /// ``` pub fn render(&mut self, f: &mut Frame, area: Rect, app: &AppState, theme: &Theme) { let block = Block::default() .borders(Borders::ALL) diff --git a/src/tui/components/help_screen.rs b/src/tui/components/help_screen.rs index 8cfe995..523ebf8 100644 --- a/src/tui/components/help_screen.rs +++ b/src/tui/components/help_screen.rs @@ -9,9 +9,37 @@ use ratatui::{ use crate::tui::{keybindings::KeyBindings, theme::Theme}; +/// Help screen rendering. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::HelpScreen, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| HelpScreen::render(f, f.area(), &theme)) +/// .unwrap(); +/// ``` pub struct HelpScreen; impl HelpScreen { + /// Render the help screen. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::HelpScreen, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| HelpScreen::render(f, f.area(), &theme)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, theme: &Theme) { let block = Block::default() .borders(Borders::ALL) diff --git a/src/tui/components/history_panel.rs b/src/tui/components/history_panel.rs index 273f310..b53f6f2 100644 --- a/src/tui/components/history_panel.rs +++ b/src/tui/components/history_panel.rs @@ -12,9 +12,39 @@ use crate::tui::{ theme::Theme, }; +/// Conversion history panel rendering. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::HistoryPanel, state::AppState, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let app = AppState::new(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| HistoryPanel::render(f, f.area(), &app, &theme)) +/// .unwrap(); +/// ``` pub struct HistoryPanel; impl HistoryPanel { + /// Render the conversion history panel. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::HistoryPanel, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let app = AppState::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| HistoryPanel::render(f, f.area(), &app, &theme)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, app: &AppState, theme: &Theme) { let block = Block::default() .borders(Borders::ALL) diff --git a/src/tui/components/repl_panel.rs b/src/tui/components/repl_panel.rs index 5acb36c..8fa4f3e 100644 --- a/src/tui/components/repl_panel.rs +++ b/src/tui/components/repl_panel.rs @@ -8,9 +8,37 @@ use ratatui::{ use crate::tui::state::{AppState, ReplLineKind}; +/// REPL panel rendering. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::ReplPanel, state::AppState}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let mut app = AppState::new(); +/// terminal +/// .draw(|f| ReplPanel::render(f, f.area(), &mut app)) +/// .unwrap(); +/// ``` pub struct ReplPanel; impl ReplPanel { + /// Render the REPL panel. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::ReplPanel, state::AppState}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let mut app = AppState::new(); + /// terminal + /// .draw(|f| ReplPanel::render(f, f.area(), &mut app)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, app: &mut AppState) { let chunks = Layout::default() .direction(Direction::Vertical) diff --git a/src/tui/components/settings_panel.rs b/src/tui/components/settings_panel.rs index a7debad..f4855e0 100644 --- a/src/tui/components/settings_panel.rs +++ b/src/tui/components/settings_panel.rs @@ -12,9 +12,39 @@ use crate::{ types::{Delimiter, Indent, KeyFoldingMode, PathExpansionMode}, }; +/// Settings panel rendering. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::SettingsPanel, state::AppState, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let app = AppState::new(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| SettingsPanel::render(f, f.area(), &app, &theme)) +/// .unwrap(); +/// ``` pub struct SettingsPanel; impl SettingsPanel { + /// Render the settings panel. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::SettingsPanel, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let app = AppState::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| SettingsPanel::render(f, f.area(), &app, &theme)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, app: &AppState, theme: &Theme) { let block = Block::default() .borders(Borders::ALL) diff --git a/src/tui/components/stats_bar.rs b/src/tui/components/stats_bar.rs index 4afb0e8..573d101 100644 --- a/src/tui/components/stats_bar.rs +++ b/src/tui/components/stats_bar.rs @@ -9,9 +9,39 @@ use ratatui::{ use crate::tui::{state::AppState, theme::Theme}; +/// Statistics bar rendering. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::StatsBar, state::AppState, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let app = AppState::new(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| StatsBar::render(f, f.area(), &app, &theme)) +/// .unwrap(); +/// ``` pub struct StatsBar; impl StatsBar { + /// Render the statistics bar. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::StatsBar, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let app = AppState::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| StatsBar::render(f, f.area(), &app, &theme)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, app: &AppState, theme: &Theme) { if let Some(ref stats) = app.stats { let spans = vec![ diff --git a/src/tui/components/status_bar.rs b/src/tui/components/status_bar.rs index 999f650..2229b71 100644 --- a/src/tui/components/status_bar.rs +++ b/src/tui/components/status_bar.rs @@ -9,9 +9,39 @@ use ratatui::{ use crate::tui::{state::AppState, theme::Theme}; +/// Status bar rendering. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{components::StatusBar, state::AppState, theme::Theme}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let app = AppState::new(); +/// let theme = Theme::default(); +/// terminal +/// .draw(|f| StatusBar::render(f, f.area(), &app, &theme)) +/// .unwrap(); +/// ``` pub struct StatusBar; impl StatusBar { + /// Render the status bar. + /// + /// # Examples + /// ```no_run + /// use ratatui::{backend::TestBackend, Terminal}; + /// use toon_format::tui::{components::StatusBar, state::AppState, theme::Theme}; + /// + /// let backend = TestBackend::new(80, 24); + /// let mut terminal = Terminal::new(backend).unwrap(); + /// let app = AppState::new(); + /// let theme = Theme::default(); + /// terminal + /// .draw(|f| StatusBar::render(f, f.area(), &app, &theme)) + /// .unwrap(); + /// ``` pub fn render(f: &mut Frame, area: Rect, app: &AppState, theme: &Theme) { let chunks = Layout::default() .direction(Direction::Horizontal) diff --git a/src/tui/events.rs b/src/tui/events.rs index 0d22c91..df5ed63 100644 --- a/src/tui/events.rs +++ b/src/tui/events.rs @@ -5,16 +5,40 @@ use std::time::Duration; use crossterm::event::{self, Event as CrosstermEvent, KeyEvent}; /// TUI events. +/// +/// # Examples +/// ```no_run +/// use toon_format::tui::events::Event; +/// +/// let _event = Event::Tick; +/// ``` pub enum Event { Key(KeyEvent), Tick, Resize, } +/// Event polling helper for the TUI. +/// +/// # Examples +/// ```no_run +/// use toon_format::tui::events::EventHandler; +/// use std::time::Duration; +/// +/// let _ = EventHandler::poll(Duration::from_millis(10)); +/// ``` pub struct EventHandler; impl EventHandler { /// Poll for next event with timeout. + /// + /// # Examples + /// ```no_run + /// use toon_format::tui::events::EventHandler; + /// use std::time::Duration; + /// + /// let _ = EventHandler::poll(Duration::from_millis(10)); + /// ``` pub fn poll(timeout: Duration) -> std::io::Result> { if event::poll(timeout)? { match event::read()? { diff --git a/src/tui/keybindings.rs b/src/tui/keybindings.rs index f755e02..28b83fc 100644 --- a/src/tui/keybindings.rs +++ b/src/tui/keybindings.rs @@ -3,6 +3,14 @@ use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; /// Actions that can be triggered by keyboard shortcuts. +/// +/// # Examples +/// ```no_run +/// use toon_format::tui::keybindings::Action; +/// +/// let action = Action::Quit; +/// let _ = action; +/// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Action { Quit, @@ -27,10 +35,29 @@ pub enum Action { None, } +/// Mapping between key events and actions. +/// +/// # Examples +/// ```no_run +/// use toon_format::tui::keybindings::KeyBindings; +/// use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; +/// +/// let action = KeyBindings::handle(KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL)); +/// let _ = action; +/// ``` pub struct KeyBindings; impl KeyBindings { /// Map key event to action. + /// + /// # Examples + /// ```no_run + /// use toon_format::tui::keybindings::KeyBindings; + /// use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; + /// + /// let action = KeyBindings::handle(KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL)); + /// let _ = action; + /// ``` pub fn handle(key: KeyEvent) -> Action { match (key.code, key.modifiers) { (KeyCode::Char('c'), KeyModifiers::CONTROL) => Action::Quit, @@ -59,6 +86,14 @@ impl KeyBindings { } /// Get list of shortcuts for help display. + /// + /// # Examples + /// ``` + /// use toon_format::tui::keybindings::KeyBindings; + /// + /// let shortcuts = KeyBindings::shortcuts(); + /// assert!(!shortcuts.is_empty()); + /// ``` pub fn shortcuts() -> Vec<(&'static str, &'static str)> { vec![ ("Ctrl+C/Q", "Quit"), diff --git a/src/tui/mod.rs b/src/tui/mod.rs index a7037ed..7fba5b2 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -25,6 +25,13 @@ use ratatui::{backend::CrosstermBackend, Terminal}; /// Initialize and run the TUI application. /// /// Sets up terminal in raw mode, runs the app, then restores terminal state. +/// +/// # Examples +/// ```no_run +/// use toon_format::tui::run; +/// +/// run().unwrap(); +/// ``` pub fn run() -> Result<()> { enable_raw_mode()?; let mut stdout = io::stdout(); diff --git a/src/tui/repl_command.rs b/src/tui/repl_command.rs index 33e15ee..d27e8a7 100644 --- a/src/tui/repl_command.rs +++ b/src/tui/repl_command.rs @@ -2,7 +2,15 @@ use anyhow::{bail, Result}; -/// Parsed REPL command with inline data +/// Parsed REPL command with inline data. +/// +/// # Examples +/// ``` +/// use toon_format::tui::repl_command::ReplCommand; +/// +/// let cmd = ReplCommand::parse("encode {\"a\": 1}").unwrap(); +/// assert_eq!(cmd.name, "encode"); +/// ``` #[derive(Debug, Clone)] pub struct ReplCommand { pub name: String, @@ -17,6 +25,14 @@ impl ReplCommand { /// - `encode {"data": true}` - JSON inline /// - `decode name: Alice` - TOON inline /// - `encode $var` - Variable reference + /// + /// # Examples + /// ``` + /// use toon_format::tui::repl_command::ReplCommand; + /// + /// let cmd = ReplCommand::parse("decode name: Alice").unwrap(); + /// assert_eq!(cmd.inline_data, Some("name: Alice".to_string())); + /// ``` pub fn parse(input: &str) -> Result { let input = input.trim(); if input.is_empty() { @@ -74,10 +90,28 @@ impl ReplCommand { }) } + /// Return true if the command includes the given flag. + /// + /// # Examples + /// ``` + /// use toon_format::tui::repl_command::ReplCommand; + /// + /// let cmd = ReplCommand::parse("encode {\"a\": 1} --fold-keys").unwrap(); + /// assert!(cmd.has_flag("--fold-keys")); + /// ``` pub fn has_flag(&self, flag: &str) -> bool { self.args.iter().any(|a| a == flag) } + /// Return the value for an option when present. + /// + /// # Examples + /// ``` + /// use toon_format::tui::repl_command::ReplCommand; + /// + /// let cmd = ReplCommand::parse("encode {\"a\": 1} --indent 2").unwrap(); + /// assert_eq!(cmd.get_option("--indent"), Some("2")); + /// ``` pub fn get_option(&self, option: &str) -> Option<&str> { self.args .iter() diff --git a/src/tui/state/app_state.rs b/src/tui/state/app_state.rs index 1ef8408..9fb4d97 100644 --- a/src/tui/state/app_state.rs +++ b/src/tui/state/app_state.rs @@ -7,6 +7,14 @@ use crate::{ }; /// Conversion mode (encode/decode). +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::app_state::Mode; +/// +/// let mode = Mode::Encode; +/// assert_eq!(mode.short_name(), "Encode"); +/// ``` #[derive(Debug, Clone, Copy, PartialEq)] pub enum Mode { Encode, @@ -14,6 +22,15 @@ pub enum Mode { } impl Mode { + /// Toggle between encode and decode. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::app_state::Mode; + /// + /// let mode = Mode::Encode.toggle(); + /// assert_eq!(mode, Mode::Decode); + /// ``` pub fn toggle(&self) -> Self { match self { Mode::Encode => Mode::Decode, @@ -21,6 +38,14 @@ impl Mode { } } + /// Return the full display name for the mode. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::app_state::Mode; + /// + /// assert!(Mode::Encode.as_str().contains("Encode")); + /// ``` pub fn as_str(&self) -> &'static str { match self { Mode::Encode => "Encode (JSON → TOON)", @@ -28,6 +53,14 @@ impl Mode { } } + /// Return a short display name for the mode. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::app_state::Mode; + /// + /// assert_eq!(Mode::Decode.short_name(), "Decode"); + /// ``` pub fn short_name(&self) -> &'static str { match self { Mode::Encode => "Encode", @@ -37,6 +70,21 @@ impl Mode { } /// Statistics from the last conversion. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::app_state::ConversionStats; +/// +/// let stats = ConversionStats { +/// json_tokens: 1, +/// toon_tokens: 1, +/// json_bytes: 1, +/// toon_bytes: 1, +/// token_savings: 0.0, +/// byte_savings: 0.0, +/// }; +/// let _ = stats; +/// ``` #[derive(Debug, Clone)] pub struct ConversionStats { pub json_tokens: usize, @@ -48,6 +96,14 @@ pub struct ConversionStats { } /// Central application state containing all UI and conversion state. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::AppState; +/// +/// let state = AppState::new(); +/// let _ = state; +/// ``` pub struct AppState<'a> { pub mode: Mode, pub editor: EditorState<'a>, @@ -68,6 +124,15 @@ pub struct AppState<'a> { } impl<'a> AppState<'a> { + /// Create a new application state with default values. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let state = AppState::new(); + /// let _ = state; + /// ``` pub fn new() -> Self { Self { mode: Mode::Encode, @@ -93,39 +158,111 @@ impl<'a> AppState<'a> { } } + /// Toggle between encode and decode modes. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_mode(); + /// ``` pub fn toggle_mode(&mut self) { self.mode = self.mode.toggle(); self.clear_error(); self.clear_status(); } + /// Toggle the active theme. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_theme(); + /// ``` pub fn toggle_theme(&mut self) { self.theme = self.theme.toggle(); self.set_status("Theme toggled".to_string()); } + /// Set an error message and clear the status. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.set_error("Oops".to_string()); + /// ``` pub fn set_error(&mut self, msg: String) { self.error_message = Some(msg); self.status_message = None; } + /// Set a status message and clear the error. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.set_status("OK".to_string()); + /// ``` pub fn set_status(&mut self, msg: String) { self.status_message = Some(msg); self.error_message = None; } + /// Clear the current error message. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.clear_error(); + /// ``` pub fn clear_error(&mut self) { self.error_message = None; } + /// Clear the current status message. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.clear_status(); + /// ``` pub fn clear_status(&mut self) { self.status_message = None; } + /// Mark the application to quit. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.quit(); + /// ``` pub fn quit(&mut self) { self.should_quit = true; } + /// Toggle the settings panel. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_settings(); + /// ``` pub fn toggle_settings(&mut self) { self.show_settings = !self.show_settings; if self.show_settings { @@ -136,6 +273,15 @@ impl<'a> AppState<'a> { } } + /// Toggle the help overlay. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_help(); + /// ``` pub fn toggle_help(&mut self) { self.show_help = !self.show_help; if self.show_help { @@ -146,6 +292,15 @@ impl<'a> AppState<'a> { } } + /// Toggle the file browser panel. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_file_browser(); + /// ``` pub fn toggle_file_browser(&mut self) { self.show_file_browser = !self.show_file_browser; if self.show_file_browser { @@ -156,6 +311,15 @@ impl<'a> AppState<'a> { } } + /// Toggle the conversion history panel. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_history(); + /// ``` pub fn toggle_history(&mut self) { self.show_history = !self.show_history; if self.show_history { @@ -166,6 +330,15 @@ impl<'a> AppState<'a> { } } + /// Toggle the diff panel. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_diff(); + /// ``` pub fn toggle_diff(&mut self) { self.show_diff = !self.show_diff; if self.show_diff { @@ -176,6 +349,15 @@ impl<'a> AppState<'a> { } } + /// Cycle the active encode delimiter. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.cycle_delimiter(); + /// ``` pub fn cycle_delimiter(&mut self) { self.encode_options.delimiter = match self.encode_options.delimiter { Delimiter::Comma => Delimiter::Tab, @@ -184,6 +366,15 @@ impl<'a> AppState<'a> { }; } + /// Increase indentation (up to the maximum). + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.increase_indent(); + /// ``` pub fn increase_indent(&mut self) { let Indent::Spaces(current) = self.encode_options.indent; if current < 8 { @@ -191,6 +382,15 @@ impl<'a> AppState<'a> { } } + /// Decrease indentation (down to the minimum). + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.decrease_indent(); + /// ``` pub fn decrease_indent(&mut self) { let Indent::Spaces(current) = self.encode_options.indent; if current > 1 { @@ -198,6 +398,15 @@ impl<'a> AppState<'a> { } } + /// Toggle key folding on/off. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_fold_keys(); + /// ``` pub fn toggle_fold_keys(&mut self) { self.encode_options.key_folding = match self.encode_options.key_folding { KeyFoldingMode::Off => KeyFoldingMode::Safe, @@ -205,6 +414,15 @@ impl<'a> AppState<'a> { }; } + /// Increase the flatten depth used for key folding. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.increase_flatten_depth(); + /// ``` pub fn increase_flatten_depth(&mut self) { if self.encode_options.flatten_depth == usize::MAX { self.encode_options.flatten_depth = 2; @@ -213,6 +431,15 @@ impl<'a> AppState<'a> { } } + /// Decrease the flatten depth used for key folding. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.decrease_flatten_depth(); + /// ``` pub fn decrease_flatten_depth(&mut self) { if self.encode_options.flatten_depth == 2 { self.encode_options.flatten_depth = usize::MAX; @@ -223,6 +450,15 @@ impl<'a> AppState<'a> { } } + /// Toggle key folding depth between default and minimum. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_flatten_depth(); + /// ``` pub fn toggle_flatten_depth(&mut self) { if self.encode_options.flatten_depth == usize::MAX { self.encode_options.flatten_depth = 2; @@ -231,6 +467,15 @@ impl<'a> AppState<'a> { } } + /// Toggle path expansion on/off. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_expand_paths(); + /// ``` pub fn toggle_expand_paths(&mut self) { self.decode_options.expand_paths = match self.decode_options.expand_paths { PathExpansionMode::Off => PathExpansionMode::Safe, @@ -238,10 +483,28 @@ impl<'a> AppState<'a> { }; } + /// Toggle strict mode for decoding. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_strict(); + /// ``` pub fn toggle_strict(&mut self) { self.decode_options.strict = !self.decode_options.strict; } + /// Toggle type coercion for decoding. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::AppState; + /// + /// let mut state = AppState::new(); + /// state.toggle_coerce_types(); + /// ``` pub fn toggle_coerce_types(&mut self) { self.decode_options.coerce_types = !self.decode_options.coerce_types; } diff --git a/src/tui/state/editor_state.rs b/src/tui/state/editor_state.rs index 25d2ffb..1d0ee94 100644 --- a/src/tui/state/editor_state.rs +++ b/src/tui/state/editor_state.rs @@ -3,6 +3,14 @@ use tui_textarea::TextArea; /// Which panel is currently active. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::editor_state::EditorMode; +/// +/// let mode = EditorMode::Input; +/// let _ = mode; +/// ``` #[derive(Debug, Clone, Copy, PartialEq)] pub enum EditorMode { Input, @@ -10,6 +18,14 @@ pub enum EditorMode { } /// State for input and output text areas. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::EditorState; +/// +/// let state = EditorState::new(); +/// let _ = state; +/// ``` pub struct EditorState<'a> { pub input: TextArea<'a>, pub output: TextArea<'a>, @@ -17,6 +33,15 @@ pub struct EditorState<'a> { } impl<'a> EditorState<'a> { + /// Create a new editor state with placeholder text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let state = EditorState::new(); + /// let _ = state; + /// ``` pub fn new() -> Self { let mut input = TextArea::default(); input.set_placeholder_text("Enter JSON here or open a file (Ctrl+O)"); @@ -31,36 +56,99 @@ impl<'a> EditorState<'a> { } } + /// Replace the input text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let mut state = EditorState::new(); + /// state.set_input("{}".to_string()); + /// ``` pub fn set_input(&mut self, text: String) { let lines: Vec = text.lines().map(|l| l.to_string()).collect(); self.input = TextArea::from(lines); } + /// Replace the output text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let mut state = EditorState::new(); + /// state.set_output("result".to_string()); + /// ``` pub fn set_output(&mut self, text: String) { let lines: Vec = text.lines().map(|l| l.to_string()).collect(); self.output = TextArea::from(lines); } + /// Read the current input text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let state = EditorState::new(); + /// let _ = state.get_input(); + /// ``` pub fn get_input(&self) -> String { self.input.lines().join("\n") } + /// Read the current output text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let state = EditorState::new(); + /// let _ = state.get_output(); + /// ``` pub fn get_output(&self) -> String { self.output.lines().join("\n") } + /// Clear the input editor and restore the placeholder. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let mut state = EditorState::new(); + /// state.clear_input(); + /// ``` pub fn clear_input(&mut self) { self.input = TextArea::default(); self.input .set_placeholder_text("Enter JSON here or open a file (Ctrl+O)"); } + /// Clear the output editor and restore the placeholder. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let mut state = EditorState::new(); + /// state.clear_output(); + /// ``` pub fn clear_output(&mut self) { self.output = TextArea::default(); self.output .set_placeholder_text("TOON output will appear here"); } + /// Toggle which panel is active. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let mut state = EditorState::new(); + /// state.toggle_active(); + /// ``` pub fn toggle_active(&mut self) { self.active = match self.active { EditorMode::Input => EditorMode::Output, @@ -68,10 +156,29 @@ impl<'a> EditorState<'a> { }; } + /// Return true when the input panel is active. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let state = EditorState::new(); + /// assert!(state.is_input_active()); + /// ``` pub fn is_input_active(&self) -> bool { self.active == EditorMode::Input } + /// Return true when the output panel is active. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::EditorState; + /// + /// let mut state = EditorState::new(); + /// state.toggle_active(); + /// assert!(state.is_output_active()); + /// ``` pub fn is_output_active(&self) -> bool { self.active == EditorMode::Output } diff --git a/src/tui/state/file_state.rs b/src/tui/state/file_state.rs index bdff523..ff8b716 100644 --- a/src/tui/state/file_state.rs +++ b/src/tui/state/file_state.rs @@ -5,12 +5,36 @@ use std::path::PathBuf; #[cfg(feature = "tui-time")] use chrono::{DateTime, Local}; +/// Timestamp type for UI history entries. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::now_timestamp; +/// +/// let _ = now_timestamp(); +/// ``` #[cfg(feature = "tui-time")] pub type Timestamp = DateTime; +/// Timestamp type for UI history entries. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::now_timestamp; +/// +/// let _ = now_timestamp(); +/// ``` #[cfg(not(feature = "tui-time"))] pub type Timestamp = (); +/// Return the current timestamp when supported. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::now_timestamp; +/// +/// let _ = now_timestamp(); +/// ``` pub fn now_timestamp() -> Option { #[cfg(feature = "tui-time")] { @@ -23,6 +47,15 @@ pub fn now_timestamp() -> Option { } } +/// Format a timestamp for display. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::format_timestamp; +/// +/// let out = format_timestamp(&None); +/// assert!(!out.is_empty()); +/// ``` pub fn format_timestamp(timestamp: &Option) -> String { #[cfg(feature = "tui-time")] { @@ -40,6 +73,20 @@ pub fn format_timestamp(timestamp: &Option) -> String { } /// A file or directory entry. +/// +/// # Examples +/// ``` +/// use std::path::PathBuf; +/// use toon_format::tui::state::file_state::FileEntry; +/// +/// let entry = FileEntry { +/// path: PathBuf::from("data.json"), +/// is_dir: false, +/// size: 0, +/// modified: None, +/// }; +/// let _ = entry; +/// ``` #[derive(Debug, Clone)] pub struct FileEntry { pub path: PathBuf, @@ -49,6 +96,21 @@ pub struct FileEntry { } impl FileEntry { + /// Return the file name for display. + /// + /// # Examples + /// ``` + /// use std::path::PathBuf; + /// use toon_format::tui::state::file_state::FileEntry; + /// + /// let entry = FileEntry { + /// path: PathBuf::from("data.json"), + /// is_dir: false, + /// size: 0, + /// modified: None, + /// }; + /// assert_eq!(entry.name(), "data.json"); + /// ``` pub fn name(&self) -> String { self.path .file_name() @@ -57,16 +119,61 @@ impl FileEntry { .to_string() } + /// Return true if the entry is a JSON file. + /// + /// # Examples + /// ``` + /// use std::path::PathBuf; + /// use toon_format::tui::state::file_state::FileEntry; + /// + /// let entry = FileEntry { + /// path: PathBuf::from("data.json"), + /// is_dir: false, + /// size: 0, + /// modified: None, + /// }; + /// assert!(entry.is_json()); + /// ``` pub fn is_json(&self) -> bool { !self.is_dir && self.path.extension().and_then(|e| e.to_str()) == Some("json") } + /// Return true if the entry is a TOON file. + /// + /// # Examples + /// ``` + /// use std::path::PathBuf; + /// use toon_format::tui::state::file_state::FileEntry; + /// + /// let entry = FileEntry { + /// path: PathBuf::from("data.toon"), + /// is_dir: false, + /// size: 0, + /// modified: None, + /// }; + /// assert!(entry.is_toon()); + /// ``` pub fn is_toon(&self) -> bool { !self.is_dir && self.path.extension().and_then(|e| e.to_str()) == Some("toon") } } /// Record of a conversion operation. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::ConversionHistory; +/// +/// let history = ConversionHistory { +/// timestamp: None, +/// mode: "Encode".to_string(), +/// input_file: None, +/// output_file: None, +/// token_savings: None, +/// byte_savings: None, +/// }; +/// let _ = history; +/// ``` #[derive(Debug, Clone)] pub struct ConversionHistory { pub timestamp: Option, @@ -78,6 +185,14 @@ pub struct ConversionHistory { } /// File browser and conversion history state. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::FileState; +/// +/// let state = FileState::new(); +/// let _ = state; +/// ``` pub struct FileState { pub current_file: Option, pub current_dir: PathBuf, @@ -87,6 +202,15 @@ pub struct FileState { } impl FileState { + /// Create a new file state. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::FileState; + /// + /// let state = FileState::new(); + /// let _ = state; + /// ``` pub fn new() -> Self { Self { current_file: None, @@ -97,6 +221,16 @@ impl FileState { } } + /// Set the current file and update the working directory. + /// + /// # Examples + /// ``` + /// use std::path::PathBuf; + /// use toon_format::tui::state::FileState; + /// + /// let mut state = FileState::new(); + /// state.set_current_file(PathBuf::from("data.json")); + /// ``` pub fn set_current_file(&mut self, path: PathBuf) { self.current_file = Some(path.clone()); self.current_dir = path @@ -106,15 +240,49 @@ impl FileState { self.is_modified = false; } + /// Clear the current file selection. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::FileState; + /// + /// let mut state = FileState::new(); + /// state.clear_current_file(); + /// ``` pub fn clear_current_file(&mut self) { self.current_file = None; self.is_modified = false; } + /// Mark the current file as modified. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::FileState; + /// + /// let mut state = FileState::new(); + /// state.mark_modified(); + /// ``` pub fn mark_modified(&mut self) { self.is_modified = true; } + /// Add a conversion entry to history. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::{ConversionHistory, FileState}; + /// + /// let mut state = FileState::new(); + /// state.add_to_history(ConversionHistory { + /// timestamp: None, + /// mode: "Encode".to_string(), + /// input_file: None, + /// output_file: None, + /// token_savings: None, + /// byte_savings: None, + /// }); + /// ``` pub fn add_to_history(&mut self, entry: ConversionHistory) { self.history.push(entry); if self.history.len() > 50 { @@ -122,6 +290,16 @@ impl FileState { } } + /// Toggle a file's selection in the browser. + /// + /// # Examples + /// ``` + /// use std::path::PathBuf; + /// use toon_format::tui::state::FileState; + /// + /// let mut state = FileState::new(); + /// state.toggle_file_selection(PathBuf::from("data.json")); + /// ``` pub fn toggle_file_selection(&mut self, path: PathBuf) { if let Some(pos) = self.selected_files.iter().position(|p| p == &path) { self.selected_files.remove(pos); @@ -130,10 +308,29 @@ impl FileState { } } + /// Clear all selected files. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::FileState; + /// + /// let mut state = FileState::new(); + /// state.clear_selection(); + /// ``` pub fn clear_selection(&mut self) { self.selected_files.clear(); } + /// Return true if the path is selected. + /// + /// # Examples + /// ``` + /// use std::path::PathBuf; + /// use toon_format::tui::state::FileState; + /// + /// let state = FileState::new(); + /// let _ = state.is_selected(&PathBuf::from("data.json")); + /// ``` pub fn is_selected(&self, path: &PathBuf) -> bool { self.selected_files.contains(path) } diff --git a/src/tui/state/repl_state.rs b/src/tui/state/repl_state.rs index b8b6cd5..68b73e3 100644 --- a/src/tui/state/repl_state.rs +++ b/src/tui/state/repl_state.rs @@ -2,7 +2,15 @@ use std::collections::HashMap; -/// REPL session state +/// REPL session state. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::ReplState; +/// +/// let state = ReplState::new(); +/// let _ = state; +/// ``` #[derive(Debug, Clone)] pub struct ReplState { /// Whether REPL is active @@ -23,13 +31,33 @@ pub struct ReplState { pub scroll_offset: usize, } -/// A line in the REPL output +/// A line in the REPL output. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::{ReplLine, ReplLineKind}; +/// +/// let line = ReplLine { +/// kind: ReplLineKind::Info, +/// content: "hello".to_string(), +/// }; +/// let _ = line; +/// ``` #[derive(Debug, Clone)] pub struct ReplLine { pub kind: ReplLineKind, pub content: String, } +/// Classification of REPL output lines. +/// +/// # Examples +/// ``` +/// use toon_format::tui::state::ReplLineKind; +/// +/// let kind = ReplLineKind::Success; +/// let _ = kind; +/// ``` #[derive(Debug, Clone, PartialEq)] pub enum ReplLineKind { Prompt, @@ -39,6 +67,15 @@ pub enum ReplLineKind { } impl ReplState { + /// Create a new REPL state with defaults. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let state = ReplState::new(); + /// let _ = state; + /// ``` pub fn new() -> Self { Self { active: false, @@ -55,18 +92,45 @@ impl ReplState { } } + /// Activate REPL mode and reset input. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.activate(); + /// ``` pub fn activate(&mut self) { self.active = true; self.input.clear(); self.history_index = None; } + /// Deactivate REPL mode and reset input. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.deactivate(); + /// ``` pub fn deactivate(&mut self) { self.active = false; self.input.clear(); self.history_index = None; } + /// Add a prompt line to the REPL output. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_prompt("help"); + /// ``` pub fn add_prompt(&mut self, cmd: &str) { self.output.push(ReplLine { kind: ReplLineKind::Prompt, @@ -74,6 +138,15 @@ impl ReplState { }); } + /// Add a success message to the REPL output. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_success("ok".to_string()); + /// ``` pub fn add_success(&mut self, msg: String) { for line in msg.lines() { self.output.push(ReplLine { @@ -83,6 +156,15 @@ impl ReplState { } } + /// Add an error message to the REPL output. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_error("oops".to_string()); + /// ``` pub fn add_error(&mut self, msg: String) { self.output.push(ReplLine { kind: ReplLineKind::Error, @@ -90,6 +172,15 @@ impl ReplState { }); } + /// Add an informational message to the REPL output. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_info("info".to_string()); + /// ``` pub fn add_info(&mut self, msg: String) { let content = if msg.is_empty() || msg.starts_with(" ") || msg.starts_with("📖") { msg @@ -103,6 +194,15 @@ impl ReplState { }); } + /// Add a command to the history buffer. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_to_history("encode".to_string()); + /// ``` pub fn add_to_history(&mut self, cmd: String) { if cmd.trim().is_empty() { return; @@ -116,6 +216,16 @@ impl ReplState { } } + /// Move up in command history. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_to_history("encode".to_string()); + /// state.history_up(); + /// ``` pub fn history_up(&mut self) { if self.history.is_empty() { return; @@ -131,6 +241,17 @@ impl ReplState { } } + /// Move down in command history. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.add_to_history("encode".to_string()); + /// state.history_up(); + /// state.history_down(); + /// ``` pub fn history_down(&mut self) { match self.history_index { None => (), @@ -146,12 +267,30 @@ impl ReplState { } } + /// Scroll the REPL output up. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.scroll_up(); + /// ``` pub fn scroll_up(&mut self) { if self.scroll_offset > 0 { self.scroll_offset -= 1; } } + /// Scroll the REPL output down. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.scroll_down(10); + /// ``` pub fn scroll_down(&mut self, visible_lines: usize) { let max_scroll = self.output.len().saturating_sub(visible_lines); if self.scroll_offset < max_scroll { @@ -159,6 +298,15 @@ impl ReplState { } } + /// Scroll to the bottom of the REPL output. + /// + /// # Examples + /// ``` + /// use toon_format::tui::state::ReplState; + /// + /// let mut state = ReplState::new(); + /// state.scroll_to_bottom(); + /// ``` pub fn scroll_to_bottom(&mut self) { if self.output.len() <= 30 { self.scroll_offset = 0; diff --git a/src/tui/theme.rs b/src/tui/theme.rs index 3cf5e56..2e17171 100644 --- a/src/tui/theme.rs +++ b/src/tui/theme.rs @@ -3,6 +3,14 @@ use ratatui::style::{Color, Modifier, Style}; /// Available color themes. +/// +/// # Examples +/// ``` +/// use toon_format::tui::theme::Theme; +/// +/// let theme = Theme::Dark; +/// let _ = theme; +/// ``` #[derive(Debug, Clone, Copy, PartialEq, Default)] pub enum Theme { #[default] @@ -12,6 +20,14 @@ pub enum Theme { impl Theme { /// Switch between dark and light themes. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark.toggle(); + /// let _ = theme; + /// ``` pub fn toggle(&self) -> Self { match self { Theme::Dark => Theme::Light, @@ -19,6 +35,15 @@ impl Theme { } } + /// Background color for the theme. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.background(); + /// ``` pub fn background(&self) -> Color { match self { Theme::Dark => Color::Black, @@ -26,6 +51,15 @@ impl Theme { } } + /// Foreground color for the theme. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.foreground(); + /// ``` pub fn foreground(&self) -> Color { match self { Theme::Dark => Color::White, @@ -33,6 +67,15 @@ impl Theme { } } + /// Border color for inactive blocks. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.border(); + /// ``` pub fn border(&self) -> Color { match self { Theme::Dark => Color::Cyan, @@ -40,6 +83,15 @@ impl Theme { } } + /// Border color for active blocks. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.border_active(); + /// ``` pub fn border_active(&self) -> Color { match self { Theme::Dark => Color::Green, @@ -47,6 +99,15 @@ impl Theme { } } + /// Title color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.title(); + /// ``` pub fn title(&self) -> Color { match self { Theme::Dark => Color::Yellow, @@ -54,22 +115,67 @@ impl Theme { } } + /// Success color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.success(); + /// ``` pub fn success(&self) -> Color { Color::Green } + /// Error color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.error(); + /// ``` pub fn error(&self) -> Color { Color::Red } + /// Warning color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.warning(); + /// ``` pub fn warning(&self) -> Color { Color::Yellow } + /// Info color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.info(); + /// ``` pub fn info(&self) -> Color { Color::Cyan } + /// Highlight color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.highlight(); + /// ``` pub fn highlight(&self) -> Color { match self { Theme::Dark => Color::Blue, @@ -77,6 +183,15 @@ impl Theme { } } + /// Selection color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.selection(); + /// ``` pub fn selection(&self) -> Color { match self { Theme::Dark => Color::DarkGray, @@ -84,6 +199,15 @@ impl Theme { } } + /// Line number color. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.line_number(); + /// ``` pub fn line_number(&self) -> Color { match self { Theme::Dark => Color::DarkGray, @@ -91,11 +215,28 @@ impl Theme { } } + /// Style for normal text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.normal_style(); + /// ``` pub fn normal_style(&self) -> Style { Style::default().fg(self.foreground()).bg(self.background()) } /// Get border style, highlighted if active. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.border_style(true); + /// ``` pub fn border_style(&self, active: bool) -> Style { Style::default().fg(if active { self.border_active() @@ -104,16 +245,43 @@ impl Theme { }) } + /// Style for titles. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.title_style(); + /// ``` pub fn title_style(&self) -> Style { Style::default() .fg(self.title()) .add_modifier(Modifier::BOLD) } + /// Style for highlighted text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.highlight_style(); + /// ``` pub fn highlight_style(&self) -> Style { Style::default().fg(self.foreground()).bg(self.highlight()) } + /// Style for selection text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.selection_style(); + /// ``` pub fn selection_style(&self) -> Style { Style::default() .fg(self.foreground()) @@ -121,28 +289,73 @@ impl Theme { .add_modifier(Modifier::BOLD) } + /// Style for error text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.error_style(); + /// ``` pub fn error_style(&self) -> Style { Style::default() .fg(self.error()) .add_modifier(Modifier::BOLD) } + /// Style for success text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.success_style(); + /// ``` pub fn success_style(&self) -> Style { Style::default() .fg(self.success()) .add_modifier(Modifier::BOLD) } + /// Style for warning text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.warning_style(); + /// ``` pub fn warning_style(&self) -> Style { Style::default() .fg(self.warning()) .add_modifier(Modifier::BOLD) } + /// Style for informational text. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.info_style(); + /// ``` pub fn info_style(&self) -> Style { Style::default().fg(self.info()) } + /// Style for line numbers. + /// + /// # Examples + /// ``` + /// use toon_format::tui::theme::Theme; + /// + /// let theme = Theme::Dark; + /// let _ = theme.line_number_style(); + /// ``` pub fn line_number_style(&self) -> Style { Style::default().fg(self.line_number()) } diff --git a/src/tui/ui.rs b/src/tui/ui.rs index 22e3ae2..f467e52 100644 --- a/src/tui/ui.rs +++ b/src/tui/ui.rs @@ -16,6 +16,20 @@ use super::{ use crate::types::{KeyFoldingMode, PathExpansionMode}; /// Main render function - orchestrates all UI components. +/// +/// # Examples +/// ```no_run +/// use ratatui::{backend::TestBackend, Terminal}; +/// use toon_format::tui::{ui, components::FileBrowser, state::AppState}; +/// +/// let backend = TestBackend::new(80, 24); +/// let mut terminal = Terminal::new(backend).unwrap(); +/// let mut app = AppState::new(); +/// let mut file_browser = FileBrowser::new(); +/// terminal +/// .draw(|f| ui::render(f, &mut app, &mut file_browser)) +/// .unwrap(); +/// ``` pub fn render(f: &mut Frame, app: &mut AppState, file_browser: &mut FileBrowser) { let theme = app.theme; diff --git a/src/types/delimiter.rs b/src/types/delimiter.rs index cb387a5..ea0370b 100644 --- a/src/types/delimiter.rs +++ b/src/types/delimiter.rs @@ -3,6 +3,14 @@ use std::fmt; use serde::{Deserialize, Serialize}; /// Delimiter character used to separate array elements. +/// +/// # Examples +/// ``` +/// use toon_format::Delimiter; +/// +/// let delim = Delimiter::Pipe; +/// assert_eq!(delim.as_char(), '|'); +/// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] pub enum Delimiter { #[default] @@ -13,6 +21,13 @@ pub enum Delimiter { impl Delimiter { /// Get the character representation of this delimiter. + /// + /// # Examples + /// ``` + /// use toon_format::Delimiter; + /// + /// assert_eq!(Delimiter::Comma.as_char(), ','); + /// ``` pub fn as_char(&self) -> char { match self { Delimiter::Comma => ',', @@ -21,8 +36,15 @@ impl Delimiter { } } - /// Get the string representation for metadata (empty for comma, char for - /// others). + /// Get the string representation for metadata (empty for comma, char for others). + /// + /// # Examples + /// ``` + /// use toon_format::Delimiter; + /// + /// assert_eq!(Delimiter::Comma.as_metadata_str(), ""); + /// assert_eq!(Delimiter::Tab.as_metadata_str(), "\t"); + /// ``` pub fn as_metadata_str(&self) -> &'static str { match self { Delimiter::Comma => "", @@ -32,6 +54,13 @@ impl Delimiter { } /// Parse a delimiter from a character. + /// + /// # Examples + /// ``` + /// use toon_format::Delimiter; + /// + /// assert_eq!(Delimiter::from_char('|'), Some(Delimiter::Pipe)); + /// ``` pub fn from_char(c: char) -> Option { match c { ',' => Some(Delimiter::Comma), @@ -42,6 +71,13 @@ impl Delimiter { } /// Check if the delimiter character appears in the string. + /// + /// # Examples + /// ``` + /// use toon_format::Delimiter; + /// + /// assert!(Delimiter::Comma.contains_in("a,b")); + /// ``` pub fn contains_in(&self, s: &str) -> bool { s.contains(self.as_char()) } diff --git a/src/types/errors.rs b/src/types/errors.rs index 5c3e70c..75cb1c4 100644 --- a/src/types/errors.rs +++ b/src/types/errors.rs @@ -2,9 +2,28 @@ use std::sync::Arc; use thiserror::Error; /// Result type alias for TOON operations. +/// +/// # Examples +/// ``` +/// use toon_format::types::ToonResult; +/// +/// fn run() -> ToonResult<()> { +/// Ok(()) +/// } +/// +/// let _ = run(); +/// ``` pub type ToonResult = std::result::Result; /// Errors that can occur during TOON encoding or decoding. +/// +/// # Examples +/// ``` +/// use toon_format::ToonError; +/// +/// let err = ToonError::InvalidInput("bad input".to_string()); +/// let _ = err; +/// ``` #[derive(Error, Debug, Clone, PartialEq)] pub enum ToonError { #[error("Invalid input: {0}")] @@ -51,6 +70,14 @@ pub enum ToonError { /// Contextual information for error reporting, including source location /// and suggestions. +/// +/// # Examples +/// ``` +/// use toon_format::types::ErrorContext; +/// +/// let ctx = ErrorContext::new("line: value"); +/// let _ = ctx; +/// ``` #[derive(Debug, Clone, PartialEq, Eq)] enum ErrorContextSource { Inline { @@ -172,6 +199,14 @@ fn extract_lines_to_strings( impl ErrorContext { /// Create a new error context with a source line. + /// + /// # Examples + /// ``` + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value"); + /// let _ = ctx; + /// ``` pub fn new(source_line: impl Into) -> Self { Self { source: ErrorContextSource::Inline { @@ -185,6 +220,15 @@ impl ErrorContext { } /// Add preceding context lines. + /// + /// # Examples + /// ``` + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value") + /// .with_preceding_lines(vec!["prev".to_string()]); + /// let _ = ctx; + /// ``` pub fn with_preceding_lines(mut self, lines: Vec) -> Self { self.ensure_inline(); if let ErrorContextSource::Inline { @@ -197,6 +241,15 @@ impl ErrorContext { } /// Add following context lines. + /// + /// # Examples + /// ``` + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value") + /// .with_following_lines(vec!["next".to_string()]); + /// let _ = ctx; + /// ``` pub fn with_following_lines(mut self, lines: Vec) -> Self { self.ensure_inline(); if let ErrorContextSource::Inline { @@ -209,12 +262,30 @@ impl ErrorContext { } /// Add a suggestion message to help fix the error. + /// + /// # Examples + /// ``` + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value") + /// .with_suggestion("check spacing"); + /// let _ = ctx; + /// ``` pub fn with_suggestion(mut self, suggestion: impl Into) -> Self { self.suggestion = Some(suggestion.into()); self } /// Add a column indicator (caret) pointing to the error position. + /// + /// # Examples + /// ``` + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value") + /// .with_indicator(3); + /// let _ = ctx; + /// ``` pub fn with_indicator(mut self, column: usize) -> Self { self.ensure_inline(); if let ErrorContextSource::Inline { indicator, .. } = &mut self.source { @@ -250,6 +321,16 @@ impl ErrorContext { } /// Create error context from a shared input buffer with lazy extraction. + /// + /// # Examples + /// ``` + /// use std::sync::Arc; + /// use toon_format::types::ErrorContext; + /// + /// let input: Arc = Arc::from("a: 1"); + /// let ctx = ErrorContext::from_shared_input(input, 1, 1, 0).unwrap(); + /// let _ = ctx; + /// ``` pub fn from_shared_input( input: Arc, line: usize, @@ -274,6 +355,14 @@ impl ErrorContext { /// Create error context from input string with automatic context /// extraction. + /// + /// # Examples + /// ``` + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::from_input("a: 1", 1, 1, 0).unwrap(); + /// let _ = ctx; + /// ``` pub fn from_input( input: &str, line: usize, @@ -286,6 +375,14 @@ impl ErrorContext { impl ToonError { /// Create a parse error at the given position. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// + /// let err = ToonError::parse_error(1, 2, "bad syntax"); + /// let _ = err; + /// ``` pub fn parse_error(line: usize, column: usize, message: impl Into) -> Self { ToonError::ParseError { line, @@ -296,6 +393,16 @@ impl ToonError { } /// Create a parse error with additional context information. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value"); + /// let err = ToonError::parse_error_with_context(1, 1, "bad", ctx); + /// let _ = err; + /// ``` pub fn parse_error_with_context( line: usize, column: usize, @@ -311,11 +418,27 @@ impl ToonError { } /// Create an error for an invalid character. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// + /// let err = ToonError::invalid_char('?', 3); + /// let _ = err; + /// ``` pub fn invalid_char(char: char, position: usize) -> Self { ToonError::InvalidCharacter { char, position } } /// Create an error for a type mismatch. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// + /// let err = ToonError::type_mismatch("object", "string"); + /// let _ = err; + /// ``` pub fn type_mismatch(expected: impl Into, found: impl Into) -> Self { ToonError::TypeMismatch { expected: expected.into(), @@ -324,6 +447,14 @@ impl ToonError { } /// Create an error for array length mismatch. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// + /// let err = ToonError::length_mismatch(2, 3); + /// let _ = err; + /// ``` pub fn length_mismatch(expected: usize, found: usize) -> Self { ToonError::LengthMismatch { expected, @@ -333,6 +464,16 @@ impl ToonError { } /// Create an array length mismatch error with context. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("items[2]: a,b"); + /// let err = ToonError::length_mismatch_with_context(2, 3, ctx); + /// let _ = err; + /// ``` pub fn length_mismatch_with_context( expected: usize, found: usize, @@ -346,6 +487,16 @@ impl ToonError { } /// Add context to an error if it supports it. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// use toon_format::types::ErrorContext; + /// + /// let ctx = ErrorContext::new("line: value"); + /// let err = ToonError::parse_error(1, 1, "bad").with_context(ctx); + /// let _ = err; + /// ``` pub fn with_context(self, context: ErrorContext) -> Self { match self { ToonError::ParseError { @@ -371,6 +522,15 @@ impl ToonError { } /// Add a suggestion to help fix the error. + /// + /// # Examples + /// ``` + /// use toon_format::ToonError; + /// + /// let err = ToonError::parse_error(1, 1, "bad") + /// .with_suggestion("check spacing"); + /// let _ = err; + /// ``` pub fn with_suggestion(self, suggestion: impl Into) -> Self { let suggestion = suggestion.into(); match self { diff --git a/src/types/folding.rs b/src/types/folding.rs index 04c4194..bb3febb 100644 --- a/src/types/folding.rs +++ b/src/types/folding.rs @@ -1,3 +1,12 @@ +/// Controls whether key folding is applied during encoding. +/// +/// # Examples +/// ``` +/// use toon_format::types::KeyFoldingMode; +/// +/// let mode = KeyFoldingMode::Safe; +/// let _ = mode; +/// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum KeyFoldingMode { /// No folding performed. All objects use standard nesting. @@ -7,6 +16,15 @@ pub enum KeyFoldingMode { Safe, } +/// Controls whether dotted keys are expanded during decoding. +/// +/// # Examples +/// ``` +/// use toon_format::types::PathExpansionMode; +/// +/// let mode = PathExpansionMode::Off; +/// let _ = mode; +/// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum PathExpansionMode { /// Dotted keys are treated as literal keys. No expansion. @@ -16,8 +34,15 @@ pub enum PathExpansionMode { Safe, } -/// Check if a key segment is a valid IdentifierSegment (stricter than unquoted -/// keys). +/// Check if a key segment is a valid IdentifierSegment (stricter than unquoted keys). +/// +/// # Examples +/// ``` +/// use toon_format::types::is_identifier_segment; +/// +/// assert!(is_identifier_segment("user_name")); +/// assert!(!is_identifier_segment("user.name")); +/// ``` pub fn is_identifier_segment(s: &str) -> bool { let bytes = s.as_bytes(); if bytes.is_empty() { diff --git a/src/types/options.rs b/src/types/options.rs index ec4562c..6ad722a 100644 --- a/src/types/options.rs +++ b/src/types/options.rs @@ -3,6 +3,15 @@ use crate::{ types::{Delimiter, KeyFoldingMode, PathExpansionMode}, }; +/// Indentation style used for nested structures. +/// +/// # Examples +/// ``` +/// use toon_format::Indent; +/// +/// let indent = Indent::Spaces(2); +/// let _ = indent; +/// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub enum Indent { Spaces(usize), @@ -15,6 +24,15 @@ impl Default for Indent { } impl Indent { + /// Return the indentation string for a given depth. + /// + /// # Examples + /// ``` + /// use toon_format::Indent; + /// + /// let indent = Indent::Spaces(2); + /// assert_eq!(indent.get_string(2), " "); + /// ``` pub fn get_string(&self, depth: usize) -> String { if depth == 0 { return String::new(); @@ -31,6 +49,15 @@ impl Indent { } } + /// Return the number of spaces used for indentation. + /// + /// # Examples + /// ``` + /// use toon_format::Indent; + /// + /// let indent = Indent::Spaces(4); + /// assert_eq!(indent.get_spaces(), 4); + /// ``` pub fn get_spaces(&self) -> usize { match self { Indent::Spaces(count) => *count, @@ -39,6 +66,14 @@ impl Indent { } /// Options for encoding JSON values to TOON format. +/// +/// # Examples +/// ``` +/// use toon_format::{Delimiter, EncodeOptions}; +/// +/// let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe); +/// let _ = opts; +/// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub struct EncodeOptions { pub delimiter: Delimiter, @@ -60,23 +95,55 @@ impl Default for EncodeOptions { impl EncodeOptions { /// Create new encoding options with defaults. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// + /// let opts = EncodeOptions::new(); + /// let _ = opts; + /// ``` pub fn new() -> Self { Self::default() } /// Set the delimiter for array elements. + /// + /// # Examples + /// ``` + /// use toon_format::{Delimiter, EncodeOptions}; + /// + /// let opts = EncodeOptions::new().with_delimiter(Delimiter::Tab); + /// let _ = opts; + /// ``` pub fn with_delimiter(mut self, delimiter: Delimiter) -> Self { self.delimiter = delimiter; self } /// Set the indentation string for nested structures. + /// + /// # Examples + /// ``` + /// use toon_format::{EncodeOptions, Indent}; + /// + /// let opts = EncodeOptions::new().with_indent(Indent::Spaces(4)); + /// let _ = opts; + /// ``` pub fn with_indent(mut self, style: Indent) -> Self { self.indent = style; self } /// Set indentation to a specific number of spaces. + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// + /// let opts = EncodeOptions::new().with_spaces(2); + /// let _ = opts; + /// ``` pub fn with_spaces(mut self, count: usize) -> Self { self.indent = Indent::Spaces(count); self @@ -88,6 +155,15 @@ impl EncodeOptions { /// dotted-path notation if all safety requirements are met. /// /// Default: `Off` + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::types::KeyFoldingMode; + /// + /// let opts = EncodeOptions::new().with_key_folding(KeyFoldingMode::Safe); + /// let _ = opts; + /// ``` pub fn with_key_folding(mut self, mode: KeyFoldingMode) -> Self { self.key_folding = mode; self @@ -99,6 +175,17 @@ impl EncodeOptions { /// only two-segment chains: `{a: {b: val}}` → `a.b: val`. /// /// Default: `usize::MAX` (fold entire eligible chains) + /// + /// # Examples + /// ``` + /// use toon_format::EncodeOptions; + /// use toon_format::types::KeyFoldingMode; + /// + /// let opts = EncodeOptions::new() + /// .with_key_folding(KeyFoldingMode::Safe) + /// .with_flatten_depth(2); + /// let _ = opts; + /// ``` pub fn with_flatten_depth(mut self, depth: usize) -> Self { self.flatten_depth = depth; self @@ -106,6 +193,14 @@ impl EncodeOptions { } /// Options for decoding TOON format to JSON values. +/// +/// # Examples +/// ``` +/// use toon_format::DecodeOptions; +/// +/// let opts = DecodeOptions::new().with_strict(false); +/// let _ = opts; +/// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub struct DecodeOptions { pub delimiter: Option, @@ -129,29 +224,70 @@ impl Default for DecodeOptions { impl DecodeOptions { /// Create new decoding options with defaults (strict mode enabled). + /// + /// # Examples + /// ``` + /// use toon_format::DecodeOptions; + /// + /// let opts = DecodeOptions::new(); + /// let _ = opts; + /// ``` pub fn new() -> Self { Self::default() } /// Enable or disable strict mode (validates array lengths, indentation, /// etc.). + /// + /// # Examples + /// ``` + /// use toon_format::DecodeOptions; + /// + /// let opts = DecodeOptions::new().with_strict(false); + /// let _ = opts; + /// ``` pub fn with_strict(mut self, strict: bool) -> Self { self.strict = strict; self } /// Set the expected delimiter (auto-detected if None). + /// + /// # Examples + /// ``` + /// use toon_format::{DecodeOptions, Delimiter}; + /// + /// let opts = DecodeOptions::new().with_delimiter(Delimiter::Pipe); + /// let _ = opts; + /// ``` pub fn with_delimiter(mut self, delimiter: Delimiter) -> Self { self.delimiter = Some(delimiter); self } /// Enable or disable type coercion (strings like "123" -> numbers). + /// + /// # Examples + /// ``` + /// use toon_format::DecodeOptions; + /// + /// let opts = DecodeOptions::new().with_coerce_types(false); + /// let _ = opts; + /// ``` pub fn with_coerce_types(mut self, coerce: bool) -> Self { self.coerce_types = coerce; self } + /// Set the indentation style for decode operations that require it. + /// + /// # Examples + /// ``` + /// use toon_format::{DecodeOptions, Indent}; + /// + /// let opts = DecodeOptions::new().with_indent(Indent::Spaces(2)); + /// let _ = opts; + /// ``` pub fn with_indent(mut self, style: Indent) -> Self { self.indent = style; self @@ -167,6 +303,15 @@ impl DecodeOptions { /// - `strict=false`: Last-write-wins /// /// Default: `Off` + /// + /// # Examples + /// ``` + /// use toon_format::DecodeOptions; + /// use toon_format::types::PathExpansionMode; + /// + /// let opts = DecodeOptions::new().with_expand_paths(PathExpansionMode::Safe); + /// let _ = opts; + /// ``` pub fn with_expand_paths(mut self, mode: PathExpansionMode) -> Self { self.expand_paths = mode; self diff --git a/src/types/value.rs b/src/types/value.rs index 3e55d10..f8bde9b 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -5,6 +5,15 @@ use std::{ use indexmap::IndexMap; +/// Numeric representation used by TOON. +/// +/// # Examples +/// ``` +/// use toon_format::types::Number; +/// +/// let n = Number::from(42i64); +/// assert!(n.is_i64()); +/// ``` #[derive(Clone, Debug, PartialEq)] pub enum Number { PosInt(u64), @@ -13,6 +22,15 @@ pub enum Number { } impl Number { + /// Create a floating-point number when the value is finite. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from_f64(3.14).unwrap(); + /// assert!(n.is_f64()); + /// ``` pub fn from_f64(f: f64) -> Option { if f.is_finite() { Some(Number::Float(f)) @@ -21,6 +39,15 @@ impl Number { } } + /// Returns true if the number can be represented as `i64`. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(-5i64); + /// assert!(n.is_i64()); + /// ``` pub fn is_i64(&self) -> bool { match self { Number::NegInt(_) => true, @@ -32,6 +59,15 @@ impl Number { } } + /// Returns true if the number can be represented as `u64`. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(5u64); + /// assert!(n.is_u64()); + /// ``` pub fn is_u64(&self) -> bool { match self { Number::PosInt(_) => true, @@ -43,10 +79,28 @@ impl Number { } } + /// Returns true if the number is stored as an `f64`. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(1.5f64); + /// assert!(n.is_f64()); + /// ``` pub fn is_f64(&self) -> bool { matches!(self, Number::Float(_)) } + /// Return the number as `i64` when possible. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(5i64); + /// assert_eq!(n.as_i64(), Some(5)); + /// ``` pub fn as_i64(&self) -> Option { match self { Number::PosInt(u) => { @@ -68,6 +122,15 @@ impl Number { } } + /// Return the number as `u64` when possible. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(5u64); + /// assert_eq!(n.as_u64(), Some(5)); + /// ``` pub fn as_u64(&self) -> Option { match self { Number::PosInt(u) => Some(*u), @@ -87,6 +150,15 @@ impl Number { } } + /// Return the number as `f64`. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(5u64); + /// assert_eq!(n.as_f64(), Some(5.0)); + /// ``` pub fn as_f64(&self) -> Option { match self { Number::PosInt(u) => Some(*u as f64), @@ -95,6 +167,15 @@ impl Number { } } + /// Returns true if the number has no fractional component. + /// + /// # Examples + /// ``` + /// use toon_format::types::Number; + /// + /// let n = Number::from(2.0f64); + /// assert!(n.is_integer()); + /// ``` pub fn is_integer(&self) -> bool { match self { Number::PosInt(_) | Number::NegInt(_) => true, @@ -192,8 +273,28 @@ impl From for Number { } } +/// Map type used for TOON objects. +/// +/// # Examples +/// ``` +/// use indexmap::IndexMap; +/// use toon_format::types::JsonValue; +/// +/// let mut obj: IndexMap = IndexMap::new(); +/// obj.insert("key".to_string(), JsonValue::Null); +/// assert!(obj.contains_key("key")); +/// ``` pub type Object = IndexMap; +/// TOON value representation. +/// +/// # Examples +/// ``` +/// use toon_format::types::JsonValue; +/// +/// let value = JsonValue::String("hello".to_string()); +/// assert!(value.is_string()); +/// ``` #[derive(Clone, Debug, PartialEq, Default)] pub enum JsonValue { #[default] @@ -206,31 +307,96 @@ pub enum JsonValue { } impl JsonValue { + /// Returns true if the value is `null`. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Null; + /// assert!(value.is_null()); + /// ``` pub const fn is_null(&self) -> bool { matches!(self, JsonValue::Null) } + /// Returns true if the value is a boolean. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Bool(true); + /// assert!(value.is_bool()); + /// ``` pub const fn is_bool(&self) -> bool { matches!(self, JsonValue::Bool(_)) } + /// Returns true if the value is a number. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(1u64)); + /// assert!(value.is_number()); + /// ``` pub const fn is_number(&self) -> bool { matches!(self, JsonValue::Number(_)) } + /// Returns true if the value is a string. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::String("hi".to_string()); + /// assert!(value.is_string()); + /// ``` pub const fn is_string(&self) -> bool { matches!(self, JsonValue::String(_)) } + /// Returns true if the value is an array. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Array(vec![JsonValue::Null]); + /// assert!(value.is_array()); + /// ``` pub const fn is_array(&self) -> bool { matches!(self, JsonValue::Array(_)) } + /// Returns true if the value is an object. + /// + /// # Examples + /// ``` + /// use indexmap::IndexMap; + /// use toon_format::types::JsonValue; + /// + /// let mut obj: IndexMap = IndexMap::new(); + /// obj.insert("a".to_string(), JsonValue::Null); + /// let value = JsonValue::Object(obj); + /// assert!(value.is_object()); + /// ``` pub const fn is_object(&self) -> bool { matches!(self, JsonValue::Object(_)) } - /// Returns true if the value is a number that can be represented as i64 + /// Returns true if the value is a number that can be represented as i64. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(1i64)); + /// assert!(value.is_i64()); + /// ``` pub fn is_i64(&self) -> bool { match self { JsonValue::Number(n) => n.is_i64(), @@ -238,7 +404,15 @@ impl JsonValue { } } - /// Returns true if the value is a number that can be represented as u64 + /// Returns true if the value is a number that can be represented as u64. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(1u64)); + /// assert!(value.is_u64()); + /// ``` pub fn is_u64(&self) -> bool { match self { JsonValue::Number(n) => n.is_u64(), @@ -246,6 +420,15 @@ impl JsonValue { } } + /// Returns true if the value is stored as a floating point number. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(1.5f64)); + /// assert!(value.is_f64()); + /// ``` pub fn is_f64(&self) -> bool { match self { JsonValue::Number(n) => n.is_f64(), @@ -255,6 +438,14 @@ impl JsonValue { /// If the value is a Bool, returns the associated bool. Returns None /// otherwise. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Bool(true); + /// assert_eq!(value.as_bool(), Some(true)); + /// ``` pub fn as_bool(&self) -> Option { match self { JsonValue::Bool(b) => Some(*b), @@ -264,6 +455,14 @@ impl JsonValue { /// If the value is a number, represent it as i64 if possible. Returns None /// otherwise. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(5i64)); + /// assert_eq!(value.as_i64(), Some(5)); + /// ``` pub fn as_i64(&self) -> Option { match self { JsonValue::Number(n) => n.as_i64(), @@ -273,6 +472,14 @@ impl JsonValue { /// If the value is a number, represent it as u64 if possible. Returns None /// otherwise. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(5u64)); + /// assert_eq!(value.as_u64(), Some(5)); + /// ``` pub fn as_u64(&self) -> Option { match self { JsonValue::Number(n) => n.as_u64(), @@ -282,6 +489,14 @@ impl JsonValue { /// If the value is a number, represent it as f64 if possible. Returns None /// otherwise. + /// + /// # Examples + /// ``` + /// use toon_format::types::{JsonValue, Number}; + /// + /// let value = JsonValue::Number(Number::from(5u64)); + /// assert_eq!(value.as_f64(), Some(5.0)); + /// ``` pub fn as_f64(&self) -> Option { match self { JsonValue::Number(n) => n.as_f64(), @@ -289,6 +504,15 @@ impl JsonValue { } } + /// If the value is a string, returns its contents. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::String("hi".to_string()); + /// assert_eq!(value.as_str(), Some("hi")); + /// ``` pub fn as_str(&self) -> Option<&str> { match self { JsonValue::String(s) => Some(s), @@ -296,6 +520,15 @@ impl JsonValue { } } + /// If the value is an array, returns a shared reference. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Array(vec![JsonValue::Null]); + /// assert!(value.as_array().is_some()); + /// ``` pub fn as_array(&self) -> Option<&Vec> { match self { JsonValue::Array(arr) => Some(arr), @@ -303,6 +536,15 @@ impl JsonValue { } } + /// If the value is an array, returns a mutable reference. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let mut value = JsonValue::Array(vec![JsonValue::Null]); + /// assert!(value.as_array_mut().is_some()); + /// ``` pub fn as_array_mut(&mut self) -> Option<&mut Vec> { match self { JsonValue::Array(arr) => Some(arr), @@ -310,6 +552,18 @@ impl JsonValue { } } + /// If the value is an object, returns a shared reference. + /// + /// # Examples + /// ``` + /// use indexmap::IndexMap; + /// use toon_format::types::JsonValue; + /// + /// let mut obj: IndexMap = IndexMap::new(); + /// obj.insert("a".to_string(), JsonValue::Null); + /// let value = JsonValue::Object(obj); + /// assert!(value.as_object().is_some()); + /// ``` pub fn as_object(&self) -> Option<&Object> { match self { JsonValue::Object(obj) => Some(obj), @@ -317,6 +571,18 @@ impl JsonValue { } } + /// If the value is an object, returns a mutable reference. + /// + /// # Examples + /// ``` + /// use indexmap::IndexMap; + /// use toon_format::types::JsonValue; + /// + /// let mut obj: IndexMap = IndexMap::new(); + /// obj.insert("a".to_string(), JsonValue::Null); + /// let mut value = JsonValue::Object(obj); + /// assert!(value.as_object_mut().is_some()); + /// ``` pub fn as_object_mut(&mut self) -> Option<&mut Object> { match self { JsonValue::Object(obj) => Some(obj), @@ -324,6 +590,18 @@ impl JsonValue { } } + /// Return a value by key when the value is an object. + /// + /// # Examples + /// ``` + /// use indexmap::IndexMap; + /// use toon_format::types::JsonValue; + /// + /// let mut obj: IndexMap = IndexMap::new(); + /// obj.insert("a".to_string(), JsonValue::Bool(true)); + /// let value = JsonValue::Object(obj); + /// assert!(value.get("a").is_some()); + /// ``` pub fn get(&self, key: &str) -> Option<&JsonValue> { match self { JsonValue::Object(obj) => obj.get(key), @@ -331,6 +609,15 @@ impl JsonValue { } } + /// Return a value by index when the value is an array. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Array(vec![JsonValue::Bool(true)]); + /// assert!(value.get_index(0).is_some()); + /// ``` pub fn get_index(&self, index: usize) -> Option<&JsonValue> { match self { JsonValue::Array(arr) => arr.get(index), @@ -339,10 +626,29 @@ impl JsonValue { } /// Takes the value, leaving Null in its place. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let mut value = JsonValue::Bool(true); + /// let taken = value.take(); + /// assert!(value.is_null()); + /// assert!(matches!(taken, JsonValue::Bool(true))); + /// ``` pub fn take(&mut self) -> JsonValue { std::mem::replace(self, JsonValue::Null) } + /// Return the type name used in error messages. + /// + /// # Examples + /// ``` + /// use toon_format::types::JsonValue; + /// + /// let value = JsonValue::Array(vec![]); + /// assert_eq!(value.type_name(), "array"); + /// ``` pub fn type_name(&self) -> &'static str { match self { JsonValue::Null => "null", @@ -544,7 +850,25 @@ impl From<&JsonValue> for serde_json::Value { } } +/// Convert common value types into TOON's `JsonValue`. +/// +/// # Examples +/// ``` +/// use toon_format::types::{IntoJsonValue, JsonValue}; +/// +/// let value: JsonValue = serde_json::json!({"a": 1}).into_json_value(); +/// assert!(value.is_object()); +/// ``` pub trait IntoJsonValue { + /// Convert the value into a `JsonValue`. + /// + /// # Examples + /// ``` + /// use toon_format::types::{IntoJsonValue, JsonValue}; + /// + /// let value: JsonValue = serde_json::json!({"a": 1}).into_json_value(); + /// assert!(value.is_object()); + /// ``` fn into_json_value(self) -> JsonValue; } diff --git a/src/utils/literal.rs b/src/utils/literal.rs index 7cf2c7c..3936135 100644 --- a/src/utils/literal.rs +++ b/src/utils/literal.rs @@ -1,21 +1,56 @@ use crate::constants; -/// Check if a string looks like a keyword or number (needs quoting). +/// Returns true when a string looks like a keyword or number (needs quoting). +/// +/// # Examples +/// ``` +/// use toon_format::utils::literal::is_literal_like; +/// +/// assert!(is_literal_like("null")); +/// assert!(is_literal_like("123")); +/// assert!(!is_literal_like("hello")); +/// ``` pub fn is_literal_like(s: &str) -> bool { is_keyword(s) || is_numeric_like(s) } #[inline] +/// Returns true when the string matches a reserved TOON keyword. +/// +/// # Examples +/// ``` +/// use toon_format::utils::literal::is_keyword; +/// +/// assert!(is_keyword("true")); +/// assert!(!is_keyword("TRUE")); +/// ``` pub fn is_keyword(s: &str) -> bool { constants::is_keyword(s) } #[inline] +/// Returns true when the character has structural meaning in TOON. +/// +/// # Examples +/// ``` +/// use toon_format::utils::literal::is_structural_char; +/// +/// assert!(is_structural_char('[')); +/// assert!(!is_structural_char('a')); +/// ``` pub fn is_structural_char(ch: char) -> bool { constants::is_structural_char(ch) } -/// Check if a string looks like a number (starts with digit, no leading zeros). +/// Returns true when the string looks like a number (starts with digit, no leading zeros). +/// +/// # Examples +/// ``` +/// use toon_format::utils::literal::is_numeric_like; +/// +/// assert!(is_numeric_like("3.14")); +/// assert!(!is_numeric_like("01")); +/// ``` pub fn is_numeric_like(s: &str) -> bool { let bytes = s.as_bytes(); if bytes.is_empty() { diff --git a/src/utils/mod.rs b/src/utils/mod.rs index d7d3e54..f99eee2 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -17,6 +17,14 @@ use rayon::prelude::*; use crate::types::{JsonValue as Value, Number}; /// Context for determining when quoting is needed. +/// +/// # Examples +/// ``` +/// use toon_format::utils::QuotingContext; +/// +/// let context = QuotingContext::ObjectValue; +/// let _ = context; +/// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum QuotingContext { ObjectValue, @@ -27,6 +35,16 @@ pub enum QuotingContext { const PARALLEL_THRESHOLD: usize = 256; /// Normalize a JSON value (converts NaN/Infinity to null, -0 to 0). +/// +/// # Examples +/// ``` +/// use serde_json::json; +/// use toon_format::utils::normalize; +/// +/// let value = json!(f64::NAN); +/// let normalized = normalize(value.into()); +/// assert_eq!(serde_json::Value::from(normalized), json!(null)); +/// ``` pub fn normalize(value: Value) -> Value { match value { Value::Number(n) => { diff --git a/src/utils/number.rs b/src/utils/number.rs index bb16139..2e03aa6 100644 --- a/src/utils/number.rs +++ b/src/utils/number.rs @@ -4,12 +4,32 @@ use ryu::Buffer as RyuBuffer; use crate::types::Number; /// Format a number in TOON canonical form (no exponents, no trailing zeros). +/// +/// # Examples +/// ``` +/// use toon_format::types::Number; +/// use toon_format::utils::number::format_canonical_number; +/// +/// let n = Number::from(42i64); +/// assert_eq!(format_canonical_number(&n), "42"); +/// ``` pub fn format_canonical_number(n: &Number) -> String { let mut out = String::new(); write_canonical_number_into(n, &mut out); out } +/// Write a number in TOON canonical form into a buffer. +/// +/// # Examples +/// ``` +/// use toon_format::types::Number; +/// use toon_format::utils::number::write_canonical_number_into; +/// +/// let mut out = String::new(); +/// write_canonical_number_into(&Number::from(3.14f64), &mut out); +/// assert!(out.starts_with("3.14")); +/// ``` pub fn write_canonical_number_into(n: &Number, out: &mut String) { match n { Number::PosInt(u) => write_u64(out, *u), diff --git a/src/utils/string.rs b/src/utils/string.rs index 3d1e630..6be7cff 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -1,6 +1,13 @@ use crate::{types::Delimiter, utils::literal}; /// Escape special characters in a string for quoted output. +/// +/// # Examples +/// ``` +/// use toon_format::utils::string::escape_string; +/// +/// assert_eq!(escape_string("hello\nworld"), "hello\\nworld"); +/// ``` pub fn escape_string(s: &str) -> String { let mut result = String::with_capacity(s.len()); @@ -10,6 +17,15 @@ pub fn escape_string(s: &str) -> String { } /// Escape special characters in a string and append to the output buffer. +/// +/// # Examples +/// ``` +/// use toon_format::utils::string::escape_string_into; +/// +/// let mut out = String::new(); +/// escape_string_into(&mut out, "a\tb"); +/// assert_eq!(out, "a\\tb"); +/// ``` pub fn escape_string_into(out: &mut String, s: &str) { for ch in s.chars() { match ch { @@ -38,6 +54,13 @@ pub fn escape_string_into(out: &mut String, s: &str) { /// /// Returns an error if the string contains an invalid escape sequence /// or if a backslash appears at the end of the string. +/// +/// # Examples +/// ``` +/// use toon_format::utils::string::unescape_string; +/// +/// assert_eq!(unescape_string("hello\\nworld").unwrap(), "hello\nworld"); +/// ``` pub fn unescape_string(s: &str) -> Result { let mut result = String::with_capacity(s.len()); let mut chars = s.chars().peekable(); @@ -110,13 +133,28 @@ fn is_valid_unquoted_key_internal(key: &str, allow_hyphen: bool) -> bool { }) } -/// Check if a key can be written without quotes (alphanumeric, underscore, -/// dot). +/// Check if a key can be written without quotes (alphanumeric, underscore, dot). +/// +/// # Examples +/// ``` +/// use toon_format::utils::string::is_valid_unquoted_key; +/// +/// assert!(is_valid_unquoted_key("user_name")); +/// assert!(!is_valid_unquoted_key("1bad")); +/// ``` pub fn is_valid_unquoted_key(key: &str) -> bool { is_valid_unquoted_key_internal(key, false) } /// Determine if a string needs quoting based on content and delimiter. +/// +/// # Examples +/// ``` +/// use toon_format::utils::string::needs_quoting; +/// +/// assert!(needs_quoting("true", ',')); +/// assert!(!needs_quoting("hello", ',')); +/// ``` pub fn needs_quoting(s: &str, delimiter: char) -> bool { if s.is_empty() { return true; @@ -174,6 +212,13 @@ pub fn needs_quoting(s: &str, delimiter: char) -> bool { } /// Quote and escape a string. +/// +/// # Examples +/// ``` +/// use toon_format::utils::string::quote_string; +/// +/// assert_eq!(quote_string("hello"), "\"hello\""); +/// ``` pub fn quote_string(s: &str) -> String { let mut result = String::with_capacity(s.len() + 2); result.push('"'); @@ -182,6 +227,16 @@ pub fn quote_string(s: &str) -> String { result } +/// Split a string by the active delimiter, honoring quoted segments. +/// +/// # Examples +/// ``` +/// use toon_format::types::Delimiter; +/// use toon_format::utils::string::split_by_delimiter; +/// +/// let parts = split_by_delimiter("a,\"b,c\",d", Delimiter::Comma); +/// assert_eq!(parts, vec!["a", "\"b,c\"", "d"]); +/// ``` pub fn split_by_delimiter(s: &str, delimiter: Delimiter) -> Vec { let mut result = Vec::new(); let mut current = String::new(); diff --git a/src/utils/validation.rs b/src/utils/validation.rs index f85aee9..ed6cfeb 100644 --- a/src/utils/validation.rs +++ b/src/utils/validation.rs @@ -3,6 +3,14 @@ use serde_json::Value; use crate::types::{ToonError, ToonResult}; /// Validate that nesting depth doesn't exceed the maximum. +/// +/// # Examples +/// ``` +/// use toon_format::utils::validation::validate_depth; +/// +/// assert!(validate_depth(1, 2).is_ok()); +/// assert!(validate_depth(3, 2).is_err()); +/// ``` pub fn validate_depth(depth: usize, max_depth: usize) -> ToonResult<()> { if depth > max_depth { return Err(ToonError::InvalidStructure(format!( @@ -13,6 +21,14 @@ pub fn validate_depth(depth: usize, max_depth: usize) -> ToonResult<()> { } /// Validate that a field name is not empty. +/// +/// # Examples +/// ``` +/// use toon_format::utils::validation::validate_field_name; +/// +/// assert!(validate_field_name("name").is_ok()); +/// assert!(validate_field_name("").is_err()); +/// ``` pub fn validate_field_name(name: &str) -> ToonResult<()> { if name.is_empty() { return Err(ToonError::InvalidInput( @@ -23,6 +39,15 @@ pub fn validate_field_name(name: &str) -> ToonResult<()> { } /// Recursively validate a JSON value and all nested fields. +/// +/// # Examples +/// ``` +/// use serde_json::json; +/// use toon_format::utils::validation::validate_value; +/// +/// assert!(validate_value(&json!({"name": "Ada"})).is_ok()); +/// assert!(validate_value(&json!({"": "bad"})).is_err()); +/// ``` pub fn validate_value(value: &Value) -> ToonResult<()> { match value { Value::Object(obj) => { From a8239156edb309ebbea1aa60cf2fbcef7f1ac842 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 20:12:18 +0000 Subject: [PATCH 24/24] docs: update cli help text --- src/cli/main.rs | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/cli/main.rs b/src/cli/main.rs index 5115956..6c704f5 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -36,13 +36,19 @@ EXAMPLES: toon input.json --fold-keys # Collapse {a:{b:1}} to a.b: 1 toon input.json --fold-keys --flatten-depth 2 - toon input.toon --expand-paths # Expand a.b:1 to {\"a\":{\"b\":1}}", + toon input.toon --expand-paths # Expand a.b:1 to {\"a\":{\"b\":1}} + +NOTES: + - Auto-detect uses file extension: .json -> encode, .toon -> decode. + - When reading from stdin, encode is the default (use -d to decode). + - --interactive cannot be combined with other operations. + - --stats requires the cli-stats feature.", disable_help_subcommand = true )] struct Cli { input: Option, - #[arg(short, long, help = "Launch interactive TUI mode")] + #[arg(short, long, help = "Launch interactive TUI mode (standalone)")] interactive: bool, #[arg(short, long, help = "Output file path")] @@ -54,13 +60,17 @@ struct Cli { #[arg(short, long, help = "Force decode mode (TOON → JSON)")] decode: bool, - #[arg(long, help = "Show token count and savings")] + #[arg(long, help = "Show token count and savings (encode only, cli-stats)")] stats: bool, - #[arg(long, value_parser = parse_delimiter, help = "Delimiter: comma, tab, or pipe")] + #[arg( + long, + value_parser = parse_delimiter, + help = "Delimiter: comma, tab, or pipe (encode only)" + )] delimiter: Option, - #[arg(long, value_parser = parse_indent, help = "Indentation spaces")] + #[arg(long, value_parser = parse_indent, help = "Indentation spaces (encode only)")] indent: Option, #[arg(long, help = "Disable strict validation (decode)")] @@ -69,7 +79,7 @@ struct Cli { #[arg(long, help = "Disable type coercion (decode)")] no_coerce: bool, - #[arg(long, help = "Indent output JSON with N spaces")] + #[arg(long, help = "Indent output JSON with N spaces (decode only)")] json_indent: Option, #[arg( @@ -78,7 +88,7 @@ struct Cli { )] fold_keys: bool, - #[arg(long, help = "Max depth for key folding (default: unlimited)")] + #[arg(long, help = "Max depth for key folding (encode only)")] flatten_depth: Option, #[arg(