From dc52a0dd1e45584a76af9885e2f71ee1261be153 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:52:22 +0000 Subject: [PATCH 01/11] chore: rustfmt --- examples/parts/arrays.rs | 5 +- examples/parts/arrays_of_arrays.rs | 5 +- examples/parts/decode_strict.rs | 5 +- examples/parts/delimiters.rs | 6 +- examples/parts/mixed_arrays.rs | 5 +- examples/parts/objects.rs | 5 +- examples/parts/round_trip.rs | 15 +---- examples/parts/structs.rs | 10 +-- examples/parts/tabular.rs | 5 +- src/cli/main.rs | 29 ++------ src/decode/expansion.rs | 8 +-- src/decode/mod.rs | 5 +- src/decode/parser.rs | 30 ++------- src/decode/scanner.rs | 6 +- src/decode/validation.rs | 5 +- src/encode/folding.rs | 6 +- src/encode/mod.rs | 14 +--- src/encode/writer.rs | 12 +--- src/lib.rs | 99 +++++----------------------- src/tui/app.rs | 35 ++-------- src/tui/components/diff_viewer.rs | 25 ++----- src/tui/components/editor.rs | 10 +-- src/tui/components/file_browser.rs | 26 ++------ src/tui/components/help_screen.rs | 26 ++------ src/tui/components/history_panel.rs | 26 ++------ src/tui/components/repl_panel.rs | 34 ++-------- src/tui/components/settings_panel.rs | 33 ++-------- src/tui/components/stats_bar.rs | 16 +---- src/tui/components/status_bar.rs | 24 ++----- src/tui/events.rs | 6 +- src/tui/keybindings.rs | 6 +- src/tui/mod.rs | 12 +--- src/tui/repl_command.rs | 5 +- src/tui/state/app_state.rs | 15 +---- src/tui/state/file_state.rs | 5 +- src/tui/state/mod.rs | 22 ++----- src/tui/theme.rs | 6 +- src/tui/ui.rs | 35 ++-------- src/types/delimiter.rs | 5 +- src/types/mod.rs | 24 ++----- src/types/options.rs | 6 +- src/types/value.rs | 5 +- src/utils/mod.rs | 18 +---- src/utils/string.rs | 5 +- src/utils/validation.rs | 5 +- tests/arrays.rs | 10 +-- tests/delimiters.rs | 13 +--- tests/errors.rs | 13 +--- tests/numeric.rs | 10 +-- tests/objects.rs | 10 +-- tests/real_world.rs | 10 +-- tests/round_trip.rs | 10 +-- tests/spec_fixtures.rs | 12 +--- tests/unicode.rs | 10 +-- 54 files changed, 140 insertions(+), 668 deletions(-) diff --git a/examples/parts/arrays.rs b/examples/parts/arrays.rs index 7f68638..260a301 100644 --- a/examples/parts/arrays.rs +++ b/examples/parts/arrays.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/arrays_of_arrays.rs b/examples/parts/arrays_of_arrays.rs index 5eb6ae7..bd6c319 100644 --- a/examples/parts/arrays_of_arrays.rs +++ b/examples/parts/arrays_of_arrays.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/decode_strict.rs b/examples/parts/decode_strict.rs index 14303bd..5af6bf9 100644 --- a/examples/parts/decode_strict.rs +++ b/examples/parts/decode_strict.rs @@ -1,8 +1,5 @@ use serde_json::Value; -use toon_format::{ - decode, - DecodeOptions, -}; +use toon_format::{decode, DecodeOptions}; pub fn decode_strict() { // Malformed: header says 2 rows, but only 1 provided diff --git a/examples/parts/delimiters.rs b/examples/parts/delimiters.rs index b44eeaa..338e7ee 100644 --- a/examples/parts/delimiters.rs +++ b/examples/parts/delimiters.rs @@ -1,9 +1,5 @@ use serde_json::json; -use toon_format::{ - encode, - Delimiter, - EncodeOptions, -}; +use toon_format::{encode, Delimiter, EncodeOptions}; pub fn delimiters() { let data = json!({ diff --git a/examples/parts/mixed_arrays.rs b/examples/parts/mixed_arrays.rs index c3e120b..3cf4afd 100644 --- a/examples/parts/mixed_arrays.rs +++ b/examples/parts/mixed_arrays.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/objects.rs b/examples/parts/objects.rs index d7b1bca..da6769f 100644 --- a/examples/parts/objects.rs +++ b/examples/parts/objects.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/examples/parts/round_trip.rs b/examples/parts/round_trip.rs index 5c5c3bd..b5873a8 100644 --- a/examples/parts/round_trip.rs +++ b/examples/parts/round_trip.rs @@ -1,15 +1,6 @@ -use serde::{ - Deserialize, - Serialize, -}; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct Product { diff --git a/examples/parts/structs.rs b/examples/parts/structs.rs index 7b49e48..14ec066 100644 --- a/examples/parts/structs.rs +++ b/examples/parts/structs.rs @@ -1,11 +1,5 @@ -use serde::{ - Deserialize, - Serialize, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde::{Deserialize, Serialize}; +use toon_format::{decode_default, encode_default}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct User { diff --git a/examples/parts/tabular.rs b/examples/parts/tabular.rs index 1e210ca..c93ba08 100644 --- a/examples/parts/tabular.rs +++ b/examples/parts/tabular.rs @@ -1,7 +1,4 @@ -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; use serde_json::json; use toon_format::encode_default; diff --git a/src/cli/main.rs b/src/cli/main.rs index ecc8e9f..311fb00 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -1,36 +1,17 @@ use std::{ fs, - io::{ - self, - Read, - Write, - }, - path::{ - Path, - PathBuf, - }, + io::{self, Read, Write}, + path::{Path, PathBuf}, }; -use anyhow::{ - bail, - Context, - Result, -}; +use anyhow::{bail, Context, Result}; use clap::Parser; use comfy_table::Table; use serde::Serialize; use tiktoken_rs::cl100k_base; use toon_format::{ - decode, - encode, - types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + decode, encode, + types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}, }; #[derive(Parser, Debug)] diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index cd645bc..ff31875 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -2,13 +2,7 @@ use indexmap::IndexMap; use crate::{ constants::QUOTED_KEY_MARKER, - types::{ - is_identifier_segment, - JsonValue as Value, - PathExpansionMode, - ToonError, - ToonResult, - }, + types::{is_identifier_segment, JsonValue as Value, PathExpansionMode, ToonError, ToonResult}, }; pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option> { diff --git a/src/decode/mod.rs b/src/decode/mod.rs index 574b3a7..420055c 100644 --- a/src/decode/mod.rs +++ b/src/decode/mod.rs @@ -6,10 +6,7 @@ pub mod validation; use serde_json::Value; -use crate::types::{ - DecodeOptions, - ToonResult, -}; +use crate::types::{DecodeOptions, ToonResult}; /// Decode a TOON string into any deserializable type. /// diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 0f40084..77f8fa6 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -1,29 +1,12 @@ -use serde_json::{ - Map, - Number, - Value, -}; +use serde_json::{Map, Number, Value}; use crate::{ - constants::{ - KEYWORDS, - MAX_DEPTH, - QUOTED_KEY_MARKER, - }, + constants::{KEYWORDS, MAX_DEPTH, QUOTED_KEY_MARKER}, decode::{ - scanner::{ - Scanner, - Token, - }, + scanner::{Scanner, Token}, validation, }, - types::{ - DecodeOptions, - Delimiter, - ErrorContext, - ToonError, - ToonResult, - }, + types::{DecodeOptions, Delimiter, ErrorContext, ToonError, ToonResult}, utils::validation::validate_depth, }; @@ -1441,10 +1424,7 @@ mod tests { #[test] fn test_round_trip_parentheses() { - use crate::{ - decode::decode_default, - encode::encode_default, - }; + use crate::{decode::decode_default, encode::encode_default}; let original = json!({ "message": "Mostly Functions (3 of 3)", diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 2301330..a04247e 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -1,8 +1,4 @@ -use crate::types::{ - Delimiter, - ToonError, - ToonResult, -}; +use crate::types::{Delimiter, ToonError, ToonResult}; /// Tokens produced by the scanner during lexical analysis. #[derive(Debug, Clone, PartialEq)] diff --git a/src/decode/validation.rs b/src/decode/validation.rs index 40608ef..9ff36b9 100644 --- a/src/decode/validation.rs +++ b/src/decode/validation.rs @@ -1,7 +1,4 @@ -use crate::types::{ - ToonError, - ToonResult, -}; +use crate::types::{ToonError, ToonResult}; /// Validate that array length matches expected value. pub fn validate_array_length(expected: usize, actual: usize) -> ToonResult<()> { diff --git a/src/encode/folding.rs b/src/encode/folding.rs index beec7ae..0034970 100644 --- a/src/encode/folding.rs +++ b/src/encode/folding.rs @@ -1,8 +1,4 @@ -use crate::types::{ - is_identifier_segment, - JsonValue as Value, - KeyFoldingMode, -}; +use crate::types::{is_identifier_segment, JsonValue as Value, KeyFoldingMode}; /// Result of chain analysis for folding. pub struct FoldableChain { diff --git a/src/encode/mod.rs b/src/encode/mod.rs index 0a11987..56dd24c 100644 --- a/src/encode/mod.rs +++ b/src/encode/mod.rs @@ -7,19 +7,9 @@ use indexmap::IndexMap; use crate::{ constants::MAX_DEPTH, types::{ - EncodeOptions, - IntoJsonValue, - JsonValue as Value, - KeyFoldingMode, - ToonError, - ToonResult, - }, - utils::{ - format_canonical_number, - normalize, - validation::validate_depth, - QuotingContext, + EncodeOptions, IntoJsonValue, JsonValue as Value, KeyFoldingMode, ToonError, ToonResult, }, + utils::{format_canonical_number, normalize, validation::validate_depth, QuotingContext}, }; /// Encode any serializable value to TOON format. diff --git a/src/encode/writer.rs b/src/encode/writer.rs index ba8ad96..7edf35e 100644 --- a/src/encode/writer.rs +++ b/src/encode/writer.rs @@ -1,15 +1,7 @@ use crate::{ - types::{ - Delimiter, - EncodeOptions, - ToonResult, - }, + types::{Delimiter, EncodeOptions, ToonResult}, utils::{ - string::{ - is_valid_unquoted_key, - needs_quoting, - quote_string, - }, + string::{is_valid_unquoted_key, needs_quoting, quote_string}, QuotingContext, }, }; diff --git a/src/lib.rs b/src/lib.rs index 67c0133..ce59d22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,66 +34,27 @@ pub mod types; pub mod utils; pub use decode::{ - decode, - decode_default, - decode_no_coerce, - decode_no_coerce_with_options, - decode_strict, + decode, decode_default, decode_no_coerce, decode_no_coerce_with_options, decode_strict, decode_strict_with_options, }; -pub use encode::{ - encode, - encode_array, - encode_default, - encode_object, -}; -pub use types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - ToonError, -}; +pub use encode::{encode, encode_array, encode_default, encode_object}; +pub use types::{DecodeOptions, Delimiter, EncodeOptions, Indent, ToonError}; pub use utils::{ - literal::{ - is_keyword, - is_literal_like, - }, + literal::{is_keyword, is_literal_like}, normalize, - string::{ - escape_string, - is_valid_unquoted_key, - needs_quoting, - }, + string::{escape_string, is_valid_unquoted_key, needs_quoting}, }; #[cfg(test)] mod tests { - use serde_json::{ - json, - Value, - }; + use serde_json::{json, Value}; use crate::{ constants::is_keyword, - decode::{ - decode_default, - decode_strict, - }, - encode::{ - encode, - encode_default, - }, - types::{ - Delimiter, - EncodeOptions, - }, - utils::{ - escape_string, - is_literal_like, - needs_quoting, - normalize, - }, + decode::{decode_default, decode_strict}, + encode::{encode, encode_default}, + types::{Delimiter, EncodeOptions}, + utils::{escape_string, is_literal_like, needs_quoting, normalize}, }; #[test] @@ -160,10 +121,7 @@ mod tests { assert!(needs_quoting("true", Delimiter::Comma.as_char())); } - use serde::{ - Deserialize, - Serialize, - }; + use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct TestUser { @@ -174,10 +132,7 @@ mod tests { #[test] fn test_encode_decode_simple_struct() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let user = TestUser { name: "Alice".to_string(), @@ -203,10 +158,7 @@ mod tests { #[test] fn test_encode_decode_with_array() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let product = TestProduct { id: 42, @@ -221,10 +173,7 @@ mod tests { #[test] fn test_encode_decode_vec_of_structs() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let users = vec![ TestUser { @@ -262,10 +211,7 @@ mod tests { #[test] fn test_encode_decode_nested_structs() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let nested = Nested { outer: OuterStruct { @@ -283,10 +229,7 @@ mod tests { #[test] fn test_round_trip_list_item_tabular_v3() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let original = json!({ "items": [ @@ -309,10 +252,7 @@ mod tests { #[test] fn test_round_trip_complex_list_item_tabular_v3() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let original = json!({ "data": [ @@ -342,10 +282,7 @@ mod tests { #[test] fn test_round_trip_mixed_list_items_v3() { - use crate::{ - decode_default, - encode_default, - }; + use crate::{decode_default, encode_default}; let original = json!({ "entries": [ diff --git a/src/tui/app.rs b/src/tui/app.rs index b5bbb6b..7b23c31 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -1,39 +1,18 @@ -use std::{ - fs, - path::PathBuf, - time::Duration, -}; +use std::{fs, path::PathBuf, time::Duration}; -use anyhow::{ - Context, - Result, -}; +use anyhow::{Context, Result}; use chrono::Local; -use crossterm::event::{ - KeyCode, - KeyEvent, -}; +use crossterm::event::{KeyCode, KeyEvent}; use tiktoken_rs::cl100k_base; use crate::{ - decode, - encode, + decode, encode, tui::{ components::FileBrowser, - events::{ - Event, - EventHandler, - }, - keybindings::{ - Action, - KeyBindings, - }, + events::{Event, EventHandler}, + keybindings::{Action, KeyBindings}, repl_command::ReplCommand, - state::{ - app_state::ConversionStats, - AppState, - ConversionHistory, - }, + state::{app_state::ConversionStats, AppState, ConversionHistory}, ui, }, }; diff --git a/src/tui/components/diff_viewer.rs b/src/tui/components/diff_viewer.rs index 516e73d..9494fb8 100644 --- a/src/tui/components/diff_viewer.rs +++ b/src/tui/components/diff_viewer.rs @@ -1,30 +1,13 @@ //! Side-by-side diff viewer for input/output comparison. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - Wrap, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Wrap}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct DiffViewer; diff --git a/src/tui/components/editor.rs b/src/tui/components/editor.rs index 6171f21..00fb043 100644 --- a/src/tui/components/editor.rs +++ b/src/tui/components/editor.rs @@ -2,17 +2,11 @@ use ratatui::{ layout::Rect, - widgets::{ - Block, - Borders, - }, + widgets::{Block, Borders}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct EditorComponent; diff --git a/src/tui/components/file_browser.rs b/src/tui/components/file_browser.rs index 52baf55..4baf080 100644 --- a/src/tui/components/file_browser.rs +++ b/src/tui/components/file_browser.rs @@ -3,31 +3,13 @@ use std::fs; use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; /// File browser state and rendering. pub struct FileBrowser { diff --git a/src/tui/components/help_screen.rs b/src/tui/components/help_screen.rs index 2f2be40..8cfe995 100644 --- a/src/tui/components/help_screen.rs +++ b/src/tui/components/help_screen.rs @@ -1,31 +1,13 @@ //! Help screen showing keyboard shortcuts. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; -use crate::tui::{ - keybindings::KeyBindings, - theme::Theme, -}; +use crate::tui::{keybindings::KeyBindings, theme::Theme}; pub struct HelpScreen; diff --git a/src/tui/components/history_panel.rs b/src/tui/components/history_panel.rs index 9bdc945..764a09d 100644 --- a/src/tui/components/history_panel.rs +++ b/src/tui/components/history_panel.rs @@ -1,31 +1,13 @@ //! Conversion history panel. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct HistoryPanel; diff --git a/src/tui/components/repl_panel.rs b/src/tui/components/repl_panel.rs index 681a566..5acb36c 100644 --- a/src/tui/components/repl_panel.rs +++ b/src/tui/components/repl_panel.rs @@ -1,36 +1,12 @@ use ratatui::{ - layout::{ - Constraint, - Direction, - Layout, - Margin, - Rect, - }, - style::{ - Color, - Modifier, - Style, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - Scrollbar, - ScrollbarOrientation, - ScrollbarState, - Wrap, - }, + layout::{Constraint, Direction, Layout, Margin, Rect}, + style::{Color, Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Scrollbar, ScrollbarOrientation, ScrollbarState, Wrap}, Frame, }; -use crate::tui::state::{ - AppState, - ReplLineKind, -}; +use crate::tui::state::{AppState, ReplLineKind}; pub struct ReplPanel; diff --git a/src/tui/components/settings_panel.rs b/src/tui/components/settings_panel.rs index c8d84c1..a7debad 100644 --- a/src/tui/components/settings_panel.rs +++ b/src/tui/components/settings_panel.rs @@ -1,38 +1,15 @@ //! Settings panel for configuring encode/decode options. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - List, - ListItem, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, Frame, }; use crate::{ - tui::{ - state::AppState, - theme::Theme, - }, - types::{ - Delimiter, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + tui::{state::AppState, theme::Theme}, + types::{Delimiter, Indent, KeyFoldingMode, PathExpansionMode}, }; pub struct SettingsPanel; diff --git a/src/tui/components/stats_bar.rs b/src/tui/components/stats_bar.rs index 3691d9b..4afb0e8 100644 --- a/src/tui/components/stats_bar.rs +++ b/src/tui/components/stats_bar.rs @@ -2,22 +2,12 @@ use ratatui::{ layout::Rect, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - }, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct StatsBar; diff --git a/src/tui/components/status_bar.rs b/src/tui/components/status_bar.rs index 8d1fb00..999f650 100644 --- a/src/tui/components/status_bar.rs +++ b/src/tui/components/status_bar.rs @@ -1,29 +1,13 @@ //! Status bar showing mode, file, and key commands. use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, Frame, }; -use crate::tui::{ - state::AppState, - theme::Theme, -}; +use crate::tui::{state::AppState, theme::Theme}; pub struct StatusBar; diff --git a/src/tui/events.rs b/src/tui/events.rs index 64a6e22..0d22c91 100644 --- a/src/tui/events.rs +++ b/src/tui/events.rs @@ -2,11 +2,7 @@ use std::time::Duration; -use crossterm::event::{ - self, - Event as CrosstermEvent, - KeyEvent, -}; +use crossterm::event::{self, Event as CrosstermEvent, KeyEvent}; /// TUI events. pub enum Event { diff --git a/src/tui/keybindings.rs b/src/tui/keybindings.rs index e82ccb1..f755e02 100644 --- a/src/tui/keybindings.rs +++ b/src/tui/keybindings.rs @@ -1,10 +1,6 @@ //! Keyboard shortcuts and action mapping. -use crossterm::event::{ - KeyCode, - KeyEvent, - KeyModifiers, -}; +use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; /// Actions that can be triggered by keyboard shortcuts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/tui/mod.rs b/src/tui/mod.rs index faa352b..a7037ed 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -18,17 +18,9 @@ use anyhow::Result; pub use app::TuiApp; use crossterm::{ execute, - terminal::{ - disable_raw_mode, - enable_raw_mode, - EnterAlternateScreen, - LeaveAlternateScreen, - }, -}; -use ratatui::{ - backend::CrosstermBackend, - Terminal, + terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}, }; +use ratatui::{backend::CrosstermBackend, Terminal}; /// Initialize and run the TUI application. /// diff --git a/src/tui/repl_command.rs b/src/tui/repl_command.rs index 5191f2f..33e15ee 100644 --- a/src/tui/repl_command.rs +++ b/src/tui/repl_command.rs @@ -1,9 +1,6 @@ //! REPL command parser with inline data support -use anyhow::{ - bail, - Result, -}; +use anyhow::{bail, Result}; /// Parsed REPL command with inline data #[derive(Debug, Clone)] diff --git a/src/tui/state/app_state.rs b/src/tui/state/app_state.rs index 7c70b32..d4925b8 100644 --- a/src/tui/state/app_state.rs +++ b/src/tui/state/app_state.rs @@ -1,20 +1,9 @@ //! Main application state. -use super::{ - EditorState, - FileState, - ReplState, -}; +use super::{EditorState, FileState, ReplState}; use crate::{ tui::theme::Theme, - types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}, }; /// Conversion mode (encode/decode). diff --git a/src/tui/state/file_state.rs b/src/tui/state/file_state.rs index 436e971..6467381 100644 --- a/src/tui/state/file_state.rs +++ b/src/tui/state/file_state.rs @@ -2,10 +2,7 @@ use std::path::PathBuf; -use chrono::{ - DateTime, - Local, -}; +use chrono::{DateTime, Local}; /// A file or directory entry. #[derive(Debug, Clone)] diff --git a/src/tui/state/mod.rs b/src/tui/state/mod.rs index 9943270..def4e5f 100644 --- a/src/tui/state/mod.rs +++ b/src/tui/state/mod.rs @@ -5,21 +5,7 @@ pub mod editor_state; pub mod file_state; pub mod repl_state; -pub use app_state::{ - AppState, - ConversionStats, - Mode, -}; -pub use editor_state::{ - EditorMode, - EditorState, -}; -pub use file_state::{ - ConversionHistory, - FileState, -}; -pub use repl_state::{ - ReplLine, - ReplLineKind, - ReplState, -}; +pub use app_state::{AppState, ConversionStats, Mode}; +pub use editor_state::{EditorMode, EditorState}; +pub use file_state::{ConversionHistory, FileState}; +pub use repl_state::{ReplLine, ReplLineKind, ReplState}; diff --git a/src/tui/theme.rs b/src/tui/theme.rs index 8d436d0..3cf5e56 100644 --- a/src/tui/theme.rs +++ b/src/tui/theme.rs @@ -1,10 +1,6 @@ //! Color themes for the TUI. -use ratatui::style::{ - Color, - Modifier, - Style, -}; +use ratatui::style::{Color, Modifier, Style}; /// Available color themes. #[derive(Debug, Clone, Copy, PartialEq, Default)] diff --git a/src/tui/ui.rs b/src/tui/ui.rs index 2910c37..22e3ae2 100644 --- a/src/tui/ui.rs +++ b/src/tui/ui.rs @@ -1,42 +1,19 @@ use ratatui::{ - layout::{ - Alignment, - Constraint, - Direction, - Layout, - Rect, - }, - text::{ - Line, - Span, - }, - widgets::{ - Block, - Borders, - Paragraph, - }, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, Frame, }; use super::{ components::{ - DiffViewer, - EditorComponent, - FileBrowser, - HelpScreen, - HistoryPanel, - ReplPanel, - SettingsPanel, - StatsBar, - StatusBar, + DiffViewer, EditorComponent, FileBrowser, HelpScreen, HistoryPanel, ReplPanel, + SettingsPanel, StatsBar, StatusBar, }, state::AppState, theme::Theme, }; -use crate::types::{ - KeyFoldingMode, - PathExpansionMode, -}; +use crate::types::{KeyFoldingMode, PathExpansionMode}; /// Main render function - orchestrates all UI components. pub fn render(f: &mut Frame, app: &mut AppState, file_browser: &mut FileBrowser) { diff --git a/src/types/delimiter.rs b/src/types/delimiter.rs index e554e6a..cb387a5 100644 --- a/src/types/delimiter.rs +++ b/src/types/delimiter.rs @@ -1,9 +1,6 @@ use std::fmt; -use serde::{ - Deserialize, - Serialize, -}; +use serde::{Deserialize, Serialize}; /// Delimiter character used to separate array elements. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] diff --git a/src/types/mod.rs b/src/types/mod.rs index 42b524f..ebfaf68 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -5,23 +5,7 @@ mod options; mod value; pub use delimiter::Delimiter; -pub use errors::{ - ErrorContext, - ToonError, - ToonResult, -}; -pub use folding::{ - is_identifier_segment, - KeyFoldingMode, - PathExpansionMode, -}; -pub use options::{ - DecodeOptions, - EncodeOptions, - Indent, -}; -pub use value::{ - IntoJsonValue, - JsonValue, - Number, -}; +pub use errors::{ErrorContext, ToonError, ToonResult}; +pub use folding::{is_identifier_segment, KeyFoldingMode, PathExpansionMode}; +pub use options::{DecodeOptions, EncodeOptions, Indent}; +pub use value::{IntoJsonValue, JsonValue, Number}; diff --git a/src/types/options.rs b/src/types/options.rs index 5e1ecb3..ec4562c 100644 --- a/src/types/options.rs +++ b/src/types/options.rs @@ -1,10 +1,6 @@ use crate::{ constants::DEFAULT_INDENT, - types::{ - Delimiter, - KeyFoldingMode, - PathExpansionMode, - }, + types::{Delimiter, KeyFoldingMode, PathExpansionMode}, }; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/types/value.rs b/src/types/value.rs index 15885cb..568302d 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -1,9 +1,6 @@ use std::{ fmt, - ops::{ - Index, - IndexMut, - }, + ops::{Index, IndexMut}, }; use indexmap::IndexMap; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index cf355c6..1b300cb 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -4,25 +4,13 @@ pub mod string; pub mod validation; use indexmap::IndexMap; -pub use literal::{ - is_keyword, - is_literal_like, - is_numeric_like, - is_structural_char, -}; +pub use literal::{is_keyword, is_literal_like, is_numeric_like, is_structural_char}; pub use number::format_canonical_number; pub use string::{ - escape_string, - is_valid_unquoted_key, - needs_quoting, - quote_string, - unescape_string, + escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, }; -use crate::types::{ - JsonValue as Value, - Number, -}; +use crate::types::{JsonValue as Value, Number}; /// Context for determining when quoting is needed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/utils/string.rs b/src/utils/string.rs index 7423d5a..3622990 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -1,7 +1,4 @@ -use crate::{ - types::Delimiter, - utils::literal, -}; +use crate::{types::Delimiter, utils::literal}; /// Escape special characters in a string for quoted output. pub fn escape_string(s: &str) -> String { diff --git a/src/utils/validation.rs b/src/utils/validation.rs index d14167b..72cdf84 100644 --- a/src/utils/validation.rs +++ b/src/utils/validation.rs @@ -1,9 +1,6 @@ use serde_json::Value; -use crate::types::{ - ToonError, - ToonResult, -}; +use crate::types::{ToonError, ToonResult}; /// Validate that nesting depth doesn't exceed the maximum. pub fn validate_depth(depth: usize, max_depth: usize) -> ToonResult<()> { diff --git a/tests/arrays.rs b/tests/arrays.rs index 437fafa..b879f52 100644 --- a/tests/arrays.rs +++ b/tests/arrays.rs @@ -1,13 +1,7 @@ use std::f64; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_tabular_arrays() { diff --git a/tests/delimiters.rs b/tests/delimiters.rs index 7d1bb28..7e52a67 100644 --- a/tests/delimiters.rs +++ b/tests/delimiters.rs @@ -1,14 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode, - encode_default, - Delimiter, - EncodeOptions, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode, encode_default, Delimiter, EncodeOptions}; #[test] fn test_delimiter_variants() { diff --git a/tests/errors.rs b/tests/errors.rs index 935b567..a100332 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -1,14 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode, - decode_default, - decode_strict, - DecodeOptions, - ToonError, -}; +use serde_json::{json, Value}; +use toon_format::{decode, decode_default, decode_strict, DecodeOptions, ToonError}; #[test] fn test_invalid_syntax_errors() { diff --git a/tests/numeric.rs b/tests/numeric.rs index 18f4679..432c43c 100644 --- a/tests/numeric.rs +++ b/tests/numeric.rs @@ -1,13 +1,7 @@ use core::f64; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_numeric_edge_cases() { diff --git a/tests/objects.rs b/tests/objects.rs index f73446b..7709b74 100644 --- a/tests/objects.rs +++ b/tests/objects.rs @@ -1,11 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_special_characters_and_quoting() { diff --git a/tests/real_world.rs b/tests/real_world.rs index d47471f..708df22 100644 --- a/tests/real_world.rs +++ b/tests/real_world.rs @@ -1,11 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_real_world_github_data() { diff --git a/tests/round_trip.rs b/tests/round_trip.rs index 3125276..c40dea0 100644 --- a/tests/round_trip.rs +++ b/tests/round_trip.rs @@ -1,13 +1,7 @@ use std::f64; -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_comprehensive_round_trips() { diff --git a/tests/spec_fixtures.rs b/tests/spec_fixtures.rs index 17c18f7..7d4a308 100644 --- a/tests/spec_fixtures.rs +++ b/tests/spec_fixtures.rs @@ -2,16 +2,8 @@ use datatest_stable::Utf8Path; use serde::Deserialize; use serde_json::Value; use toon_format::{ - decode, - encode, - types::{ - DecodeOptions, - Delimiter, - EncodeOptions, - Indent, - KeyFoldingMode, - PathExpansionMode, - }, + decode, encode, + types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode}, }; #[derive(Deserialize, Debug)] diff --git a/tests/unicode.rs b/tests/unicode.rs index 9c7f35d..447e709 100644 --- a/tests/unicode.rs +++ b/tests/unicode.rs @@ -1,11 +1,5 @@ -use serde_json::{ - json, - Value, -}; -use toon_format::{ - decode_default, - encode_default, -}; +use serde_json::{json, Value}; +use toon_format::{decode_default, encode_default}; #[test] fn test_unicode_strings() { From 8dbf842106f7442fc3be4283c96fb1390f001eae Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:54:46 +0000 Subject: [PATCH 02/11] fix(types): add safe get methods to JsonValue --- src/types/value.rs | 60 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/src/types/value.rs b/src/types/value.rs index 568302d..d81532b 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -324,6 +324,20 @@ impl JsonValue { } } + pub fn get(&self, key: &str) -> Option<&JsonValue> { + match self { + JsonValue::Object(obj) => obj.get(key), + _ => None, + } + } + + pub fn get_index(&self, index: usize) -> Option<&JsonValue> { + match self { + JsonValue::Array(arr) => arr.get(index), + _ => None, + } + } + /// Takes the value, leaving Null in its place. pub fn take(&mut self) -> JsonValue { std::mem::replace(self, JsonValue::Null) @@ -377,8 +391,16 @@ impl Index for JsonValue { fn index(&self, index: usize) -> &Self::Output { match self { - JsonValue::Array(arr) => &arr[index], - _ => panic!("cannot index into non-array value with usize"), + JsonValue::Array(arr) => arr.get(index).unwrap_or_else(|| { + panic!( + "index {index} out of bounds for array of length {}", + arr.len() + ) + }), + _ => panic!( + "cannot index into non-array value of type {}", + self.type_name() + ), } } } @@ -386,8 +408,16 @@ impl Index for JsonValue { impl IndexMut for JsonValue { fn index_mut(&mut self, index: usize) -> &mut Self::Output { match self { - JsonValue::Array(arr) => &mut arr[index], - _ => panic!("cannot index into non-array value with usize"), + JsonValue::Array(arr) => arr.get_mut(index).unwrap_or_else(|| { + panic!( + "index {index} out of bounds for array of length {}", + arr.len() + ) + }), + _ => panic!( + "cannot index into non-array value of type {}", + self.type_name() + ), } } } @@ -397,10 +427,13 @@ impl Index<&str> for JsonValue { fn index(&self, key: &str) -> &Self::Output { match self { - JsonValue::Object(obj) => obj - .get(key) - .unwrap_or_else(|| panic!("key '{key}' not found in object")), - _ => panic!("cannot index into non-object value with &str"), + JsonValue::Object(obj) => obj.get(key).unwrap_or_else(|| { + panic!("key '{key}' not found in object with {} entries", obj.len()) + }), + _ => panic!( + "cannot index into non-object value of type {}", + self.type_name() + ), } } } @@ -408,10 +441,13 @@ impl Index<&str> for JsonValue { impl IndexMut<&str> for JsonValue { fn index_mut(&mut self, key: &str) -> &mut Self::Output { match self { - JsonValue::Object(obj) => obj - .get_mut(key) - .unwrap_or_else(|| panic!("key '{key}' not found in object")), - _ => panic!("cannot index into non-object value with &str"), + JsonValue::Object(obj) => obj.get_mut(key).unwrap_or_else(|| { + panic!("key '{key}' not found in object with {} entries", obj.len()) + }), + _ => panic!( + "cannot index into non-object value of type {}", + self.type_name() + ), } } } From e69c69d2a630a194c8332681ceeaa238f063dcc6 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:00 +0000 Subject: [PATCH 03/11] fix(scanner): replace unwrap with safe option handling --- src/decode/scanner.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index a04247e..2631b88 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -348,9 +348,10 @@ impl Scanner { // Leading zeros like "05" are strings, but "0", "0.5", "-0" are numbers if s.starts_with('0') && s.len() > 1 { - let second_char = s.chars().nth(1).unwrap(); - if second_char.is_ascii_digit() { - return Ok(Token::String(s.to_string(), false)); + if let Some(second_char) = s.chars().nth(1) { + if second_char.is_ascii_digit() { + return Ok(Token::String(s.to_string(), false)); + } } } @@ -454,12 +455,14 @@ impl Scanner { _ => {} } - if trimmed.starts_with('-') || trimmed.chars().next().unwrap().is_ascii_digit() { + if trimmed.starts_with('-') || trimmed.chars().next().map_or(false, |c| c.is_ascii_digit()) + { // Leading zeros like "05" are strings if trimmed.starts_with('0') && trimmed.len() > 1 { - let second_char = trimmed.chars().nth(1).unwrap(); - if second_char.is_ascii_digit() { - return Ok(Token::String(trimmed.to_string(), false)); + if let Some(second_char) = trimmed.chars().nth(1) { + if second_char.is_ascii_digit() { + return Ok(Token::String(trimmed.to_string(), false)); + } } } From 29071099a7f4e3f78caeb99bc01eaa0abc04dcb0 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:17 +0000 Subject: [PATCH 04/11] fix(expansion): use expect with descriptive message --- src/decode/expansion.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index ff31875..3a9d080 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -84,7 +84,7 @@ pub fn deep_merge_value( } } else { target.insert(first_key.clone(), Value::Object(IndexMap::new())); - match target.get_mut(first_key).unwrap() { + match target.get_mut(first_key).expect("key was just inserted") { Value::Object(obj) => obj, _ => unreachable!(), } From 32dfc85d435ed107f277330846d490609645db24 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:37 +0000 Subject: [PATCH 05/11] fix(encode): use expect for field list lookup --- src/encode/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encode/mod.rs b/src/encode/mod.rs index 56dd24c..69b2d1b 100644 --- a/src/encode/mod.rs +++ b/src/encode/mod.rs @@ -215,7 +215,7 @@ fn write_object_impl( writer.write_newline()?; } - let value = &obj[*key]; + let value = obj.get(*key).expect("key exists in field list"); // Check if this key-value pair can be folded (v1.5 feature) // Don't fold if any sibling key is a dotted path starting with this key From 518e15c48d2a21f0c3341a4a38635b0a46debdc9 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:55:55 +0000 Subject: [PATCH 06/11] test: add panic safety tests --- tests/panic_safety.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/panic_safety.rs diff --git a/tests/panic_safety.rs b/tests/panic_safety.rs new file mode 100644 index 0000000..4fe26d8 --- /dev/null +++ b/tests/panic_safety.rs @@ -0,0 +1,27 @@ +//! Tests to verify panic safety of JsonValue operations + +use toon_format::JsonValue; + +#[test] +fn test_get_missing_key_returns_none() { + let obj = JsonValue::Object(Default::default()); + assert!(obj.get("nonexistent").is_none()); +} + +#[test] +fn test_get_on_non_object_returns_none() { + let arr = JsonValue::Array(vec![]); + assert!(arr.get("key").is_none()); +} + +#[test] +fn test_get_index_out_of_bounds_returns_none() { + let arr = JsonValue::Array(vec![]); + assert!(arr.get_index(0).is_none()); +} + +#[test] +fn test_get_index_on_non_array_returns_none() { + let obj = JsonValue::Object(Default::default()); + assert!(obj.get_index(0).is_none()); +} From 08f5d359f438abe6bea6ec4ea0553911f550d8ef Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:59:00 +0000 Subject: [PATCH 07/11] fix(types): avoid borrow overlap in IndexMut --- src/types/value.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/types/value.rs b/src/types/value.rs index d81532b..5a86e21 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -408,12 +408,12 @@ impl Index for JsonValue { impl IndexMut for JsonValue { fn index_mut(&mut self, index: usize) -> &mut Self::Output { match self { - JsonValue::Array(arr) => arr.get_mut(index).unwrap_or_else(|| { - panic!( - "index {index} out of bounds for array of length {}", - arr.len() - ) - }), + JsonValue::Array(arr) => { + let len = arr.len(); + arr.get_mut(index).unwrap_or_else(|| { + panic!("index {index} out of bounds for array of length {len}") + }) + } _ => panic!( "cannot index into non-array value of type {}", self.type_name() @@ -441,9 +441,12 @@ impl Index<&str> for JsonValue { impl IndexMut<&str> for JsonValue { fn index_mut(&mut self, key: &str) -> &mut Self::Output { match self { - JsonValue::Object(obj) => obj.get_mut(key).unwrap_or_else(|| { - panic!("key '{key}' not found in object with {} entries", obj.len()) - }), + JsonValue::Object(obj) => { + let len = obj.len(); + obj.get_mut(key).unwrap_or_else(|| { + panic!("key '{key}' not found in object with {len} entries") + }) + } _ => panic!( "cannot index into non-object value of type {}", self.type_name() From 31f75b10405d13edcd521cc2667515482c21f1a7 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 12:59:10 +0000 Subject: [PATCH 08/11] test: fix panic safety import --- tests/panic_safety.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/panic_safety.rs b/tests/panic_safety.rs index 4fe26d8..8630670 100644 --- a/tests/panic_safety.rs +++ b/tests/panic_safety.rs @@ -1,6 +1,6 @@ //! Tests to verify panic safety of JsonValue operations -use toon_format::JsonValue; +use toon_format::types::JsonValue; #[test] fn test_get_missing_key_returns_none() { From 071a27cb935712fd43e0e59534f6c7e6996cfee9 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 13:01:54 +0000 Subject: [PATCH 09/11] fix(scanner): satisfy clippy --- src/decode/scanner.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 2631b88..9ca7615 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -455,7 +455,11 @@ impl Scanner { _ => {} } - if trimmed.starts_with('-') || trimmed.chars().next().map_or(false, |c| c.is_ascii_digit()) + if trimmed.starts_with('-') + || trimmed + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) { // Leading zeros like "05" are strings if trimmed.starts_with('0') && trimmed.len() > 1 { From 67aca3ccebbc5e25f5e04901138bfc2566d6d1ed Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 14:02:12 +0000 Subject: [PATCH 10/11] fix(decoder): improve decode correctness --- src/decode/expansion.rs | 6 +- src/decode/parser.rs | 387 ++++++++++++++++++++++++++++++---------- src/decode/scanner.rs | 72 +++++--- src/types/folding.rs | 4 +- src/utils/mod.rs | 1 + src/utils/string.rs | 19 +- 6 files changed, 361 insertions(+), 128 deletions(-) diff --git a/src/decode/expansion.rs b/src/decode/expansion.rs index 3a9d080..d665d45 100644 --- a/src/decode/expansion.rs +++ b/src/decode/expansion.rs @@ -102,10 +102,8 @@ pub fn expand_paths_in_object( let mut result = IndexMap::new(); for (key, mut value) in obj { - // Expand nested objects first (depth-first) - if let Value::Object(nested_obj) = value { - value = Value::Object(expand_paths_in_object(nested_obj, mode, strict)?); - } + // Expand nested structures (arrays/objects) first (depth-first) + value = expand_paths_recursive(value, mode, strict)?; // Strip marker from quoted keys let clean_key = if key.starts_with(QUOTED_KEY_MARKER) { diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 77f8fa6..a682a9d 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -7,7 +7,7 @@ use crate::{ validation, }, types::{DecodeOptions, Delimiter, ErrorContext, ToonError, ToonResult}, - utils::validation::validate_depth, + utils::{is_valid_unquoted_key_decode, validation::validate_depth}, }; /// Context for parsing arrays to determine correct indentation depth. @@ -32,6 +32,7 @@ pub struct Parser<'a> { current_token: Token, options: DecodeOptions, delimiter: Option, + delimiter_stack: Vec>, input: &'a str, } @@ -41,12 +42,14 @@ impl<'a> Parser<'a> { let mut scanner = Scanner::new(input); let chosen_delim = options.delimiter; scanner.set_active_delimiter(chosen_delim); + scanner.set_coerce_types(options.coerce_types); let current_token = scanner.scan_token()?; Ok(Self { scanner, current_token, delimiter: chosen_delim, + delimiter_stack: Vec::new(), options, input, }) @@ -86,6 +89,87 @@ impl<'a> Parser<'a> { Ok(()) } + fn push_delimiter(&mut self, delimiter: Option) { + self.delimiter_stack.push(self.delimiter); + self.delimiter = delimiter; + self.scanner.set_active_delimiter(delimiter); + } + + fn pop_delimiter(&mut self) { + if let Some(previous) = self.delimiter_stack.pop() { + self.delimiter = previous; + self.scanner.set_active_delimiter(previous); + if let (Some(delim), Token::String(value, was_quoted)) = (previous, &self.current_token) + { + if !*was_quoted && value.len() == 1 && value.starts_with(delim.as_char()) { + self.current_token = Token::Delimiter(delim); + } + } + } + } + + fn format_key(&self, key: &str, was_quoted: bool) -> String { + if was_quoted && key.contains('.') { + format!("{QUOTED_KEY_MARKER}{key}") + } else { + key.to_string() + } + } + + fn validate_unquoted_key(&self, key: &str, was_quoted: bool) -> ToonResult<()> { + if self.options.strict && !was_quoted && !is_valid_unquoted_key_decode(key) { + return Err(self + .parse_error_with_context(format!("Invalid unquoted key: '{key}'")) + .with_suggestion("Quote the key to use special characters")); + } + Ok(()) + } + + fn validate_unquoted_string(&self, value: &str, was_quoted: bool) -> ToonResult<()> { + if self.options.strict && !was_quoted && value.contains('\t') { + return Err(self + .parse_error_with_context("Unquoted tab characters are not allowed in strict mode") + .with_suggestion("Quote the value to include tabs")); + } + Ok(()) + } + + fn is_key_token(&self) -> bool { + matches!( + self.current_token, + Token::String(_, _) | Token::Bool(_) | Token::Null + ) + } + + fn key_from_token(&self) -> Option<(String, bool)> { + match &self.current_token { + Token::String(s, was_quoted) => Some((self.format_key(s, *was_quoted), *was_quoted)), + Token::Bool(b) => Some(( + if *b { + KEYWORDS[1].to_string() + } else { + KEYWORDS[2].to_string() + }, + false, + )), + Token::Null => Some((KEYWORDS[0].to_string(), false)), + _ => None, + } + } + + fn find_unexpected_delimiter( + &self, + field: &str, + expected: Option, + ) -> Option { + let expected = expected?; + let delimiters = [Delimiter::Comma, Delimiter::Pipe, Delimiter::Tab]; + + delimiters + .into_iter() + .find(|delim| *delim != expected && field.contains(delim.as_char())) + } + fn parse_value(&mut self) -> ToonResult { self.parse_value_with_depth(0) } @@ -103,7 +187,7 @@ impl<'a> Parser<'a> { if next_char_is_colon { let key = KEYWORDS[0].to_string(); self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { self.advance()?; Ok(Value::Null) @@ -118,7 +202,7 @@ impl<'a> Parser<'a> { KEYWORDS[2].to_string() }; self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { let val = *b; self.advance()?; @@ -130,7 +214,7 @@ impl<'a> Parser<'a> { if next_char_is_colon { let key = i.to_string(); self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { let val = *i; self.advance()?; @@ -142,7 +226,7 @@ impl<'a> Parser<'a> { if next_char_is_colon { let key = n.to_string(); self.advance()?; - self.parse_object_with_initial_key(key, depth) + self.parse_object_with_initial_key(key, false, depth) } else { let val = *n; self.advance()?; @@ -158,13 +242,15 @@ impl<'a> Parser<'a> { } } } - Token::String(s, _) => { + Token::String(s, was_quoted) => { + let key_was_quoted = *was_quoted; let first = s.clone(); self.advance()?; match &self.current_token { Token::Colon | Token::LeftBracket => { - self.parse_object_with_initial_key(first, depth) + let key = self.format_key(&first, key_was_quoted); + self.parse_object_with_initial_key(key, key_was_quoted, depth) } _ => { // Strings on new indented lines could be missing colons (keys) or values @@ -189,6 +275,7 @@ impl<'a> Parser<'a> { accumulated.push_str(next); self.advance()?; } + self.validate_unquoted_string(&accumulated, key_was_quoted)?; Ok(Value::String(accumulated)) } } @@ -230,17 +317,9 @@ impl<'a> Parser<'a> { base_indent = Some(current_indent); } - let key = match &self.current_token { - Token::String(s, was_quoted) => { - // Mark quoted keys containing dots with a special prefix - // so path expansion can skip them - if *was_quoted && s.contains('.') { - format!("{QUOTED_KEY_MARKER}{s}") - } else { - s.clone() - } - } - _ => { + let (key, was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { return Err(self .parse_error_with_context(format!( "Expected key, found {:?}", @@ -249,6 +328,7 @@ impl<'a> Parser<'a> { .with_suggestion("Object keys must be strings")); } }; + self.validate_unquoted_key(&key, was_quoted)?; self.advance()?; let value = if matches!(self.current_token, Token::LeftBracket) { @@ -272,7 +352,12 @@ impl<'a> Parser<'a> { Ok(Value::Object(obj)) } - fn parse_object_with_initial_key(&mut self, key: String, depth: usize) -> ToonResult { + fn parse_object_with_initial_key( + &mut self, + key: String, + key_was_quoted: bool, + depth: usize, + ) -> ToonResult { validate_depth(depth, MAX_DEPTH)?; let mut obj = Map::new(); @@ -284,6 +369,8 @@ impl<'a> Parser<'a> { self.validate_indentation(current_indent)?; } + self.validate_unquoted_key(&key, key_was_quoted)?; + if matches!(self.current_token, Token::LeftBracket) { let value = self.parse_array(depth)?; obj.insert(key, value); @@ -335,7 +422,7 @@ impl<'a> Parser<'a> { break; } - if !matches!(self.current_token, Token::String(_, _)) { + if !self.is_key_token() { break; } @@ -365,18 +452,11 @@ impl<'a> Parser<'a> { base_indent = Some(current_indent); } - let key = match &self.current_token { - Token::String(s, was_quoted) => { - // Mark quoted keys containing dots with a special prefix - // so path expansion can skip them - if *was_quoted && s.contains('.') { - format!("{QUOTED_KEY_MARKER}{s}") - } else { - s.clone() - } - } - _ => break, + let (key, was_quoted) = match self.key_from_token() { + Some(key) => key, + None => break, }; + self.validate_unquoted_key(&key, was_quoted)?; self.advance()?; let value = if matches!(self.current_token, Token::LeftBracket) { @@ -416,12 +496,15 @@ impl<'a> Parser<'a> { self.parse_value_with_depth(depth + 1) } else { // Check if there's more content after the current token - let (rest, had_space) = self.scanner.read_rest_of_line_with_space_info(); + let (rest, leading_space) = self.scanner.read_rest_of_line_with_space_info(); let result = if rest.is_empty() { // Single token - convert directly to avoid redundant parsing match &self.current_token { - Token::String(s, _) => Ok(Value::String(s.clone())), + Token::String(s, was_quoted) => { + self.validate_unquoted_string(s, *was_quoted)?; + Ok(Value::String(s.clone())) + } Token::Integer(i) => Ok(serde_json::Number::from(*i).into()), Token::Number(n) => { let val = *n; @@ -461,14 +544,22 @@ impl<'a> Parser<'a> { } // Only add space if there was whitespace in the original input - if had_space { - value_str.push(' '); + if !rest.is_empty() { + value_str.push_str(&leading_space); } value_str.push_str(&rest); let token = self.scanner.parse_value_string(&value_str)?; match token { - Token::String(s, _) => Ok(Value::String(s)), + Token::String(s, was_quoted) => { + if self.options.strict && !was_quoted && value_str.contains('\t') { + return Err(self.parse_error_with_context( + "Unquoted tab characters are not allowed in strict mode", + )); + } + self.validate_unquoted_string(&s, was_quoted)?; + Ok(Value::String(s)) + } Token::Integer(i) => Ok(serde_json::Number::from(i).into()), Token::Number(n) => { if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 { @@ -502,9 +593,7 @@ impl<'a> Parser<'a> { self.parse_array(depth) } - fn parse_array_header( - &mut self, - ) -> ToonResult<(usize, Option, Option>)> { + fn parse_array_header(&mut self) -> ToonResult<(usize, Option, bool)> { if !matches!(self.current_token, Token::LeftBracket) { return Err(self.parse_error_with_context("Expected '['")); } @@ -559,11 +648,6 @@ impl<'a> Parser<'a> { _ => None, }; - // Default to comma if no delimiter specified - let active_delim = detected_delim.or(Some(Delimiter::Comma)); - - self.scanner.set_active_delimiter(active_delim); - if !matches!(self.current_token, Token::RightBracket) { return Err(self.parse_error_with_context(format!( "Expected ']', found {:?}", @@ -572,51 +656,114 @@ impl<'a> Parser<'a> { } self.advance()?; - let fields = if matches!(self.current_token, Token::LeftBrace) { - self.advance()?; - let mut fields = Vec::new(); + let has_fields = matches!(self.current_token, Token::LeftBrace); - loop { - match &self.current_token { - Token::String(s, _) => { - fields.push(s.clone()); - self.advance()?; + Ok((length, detected_delim, has_fields)) + } - if matches!(self.current_token, Token::RightBrace) { - break; - } + fn parse_field_list(&mut self, expected_delim: Option) -> ToonResult> { + if !matches!(self.current_token, Token::LeftBrace) { + return Err(self.parse_error_with_context("Expected '{' for field list")); + } + self.advance()?; - if matches!(self.current_token, Token::Delimiter(_)) { - self.advance()?; - } else { + let mut fields = Vec::new(); + let mut field_list_delim = None; + + loop { + match &self.current_token { + Token::String(s, was_quoted) => { + if self.options.strict { + if let Some(unexpected) = self.find_unexpected_delimiter(s, expected_delim) + { return Err(self.parse_error_with_context(format!( - "Expected delimiter or '}}', found {:?}", - self.current_token + "Field list delimiter {unexpected} does not match expected {}", + expected_delim + .map(|delim| delim.to_string()) + .unwrap_or_else(|| "none".to_string()) ))); } + self.validate_unquoted_key(s, *was_quoted)?; } - Token::RightBrace => break, - _ => { + + fields.push(self.format_key(s, *was_quoted)); + self.advance()?; + + if matches!(self.current_token, Token::RightBrace) { + break; + } + + if let Token::Delimiter(delim) = &self.current_token { + if self.options.strict { + validation::validate_delimiter_consistency( + Some(*delim), + expected_delim, + )?; + } + if field_list_delim.is_none() { + field_list_delim = Some(*delim); + } + self.advance()?; + } else { return Err(self.parse_error_with_context(format!( - "Expected field name, found {:?}", + "Expected delimiter or '}}', found {:?}", self.current_token - ))) + ))); } } - } + Token::Bool(_) | Token::Null => { + let (field, was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { + return Err(self.parse_error_with_context(format!( + "Expected field name, found {:?}", + self.current_token + ))) + } + }; + self.validate_unquoted_key(&field, was_quoted)?; + fields.push(field); + self.advance()?; - self.advance()?; - Some(fields) - } else { - None - }; + if matches!(self.current_token, Token::RightBrace) { + break; + } - if !matches!(self.current_token, Token::Colon) { - return Err(self.parse_error_with_context("Expected ':' after array header")); + if let Token::Delimiter(delim) = &self.current_token { + if self.options.strict { + validation::validate_delimiter_consistency( + Some(*delim), + expected_delim, + )?; + } + if field_list_delim.is_none() { + field_list_delim = Some(*delim); + } + self.advance()?; + } else { + return Err(self.parse_error_with_context(format!( + "Expected delimiter or '}}', found {:?}", + self.current_token + ))); + } + } + Token::RightBrace => break, + _ => { + return Err(self.parse_error_with_context(format!( + "Expected field name, found {:?}", + self.current_token + ))) + } + } } + self.advance()?; + validation::validate_field_list(&fields)?; + if self.options.strict { + validation::validate_delimiter_consistency(field_list_delim, expected_delim)?; + } - Ok((length, detected_delim, fields)) + Ok(fields) } fn parse_array(&mut self, depth: usize) -> ToonResult { @@ -630,20 +777,54 @@ impl<'a> Parser<'a> { ) -> ToonResult { validate_depth(depth, MAX_DEPTH)?; - let (length, _detected_delim, fields) = self.parse_array_header()?; + let (length, detected_delim, has_fields) = self.parse_array_header()?; - if let Some(fields) = fields { - validation::validate_field_list(&fields)?; - self.parse_tabular_array(length, &fields, depth, context) - } else { - // Non-tabular arrays as first field of list items require depth adjustment - // (items at depth +2 relative to hyphen, not the usual +1) - let adjusted_depth = match context { - ArrayParseContext::Normal => depth, - ArrayParseContext::ListItemFirstField => depth + 1, + if let (Some(detected), Some(expected)) = (detected_delim, self.options.delimiter) { + if detected != expected { + return Err(self.parse_error_with_context(format!( + "Detected delimiter {detected} but expected {expected}" + ))); + } + } + + let active_delim = detected_delim + .or(self.options.delimiter) + .or(Some(Delimiter::Comma)); + + let mut pushed = false; + let result = (|| -> ToonResult { + self.push_delimiter(active_delim); + pushed = true; + + let fields = if has_fields { + Some(self.parse_field_list(active_delim)?) + } else { + None }; - self.parse_regular_array(length, adjusted_depth) + + if !matches!(self.current_token, Token::Colon) { + return Err(self.parse_error_with_context("Expected ':' after array header")); + } + self.advance()?; + + if let Some(fields) = fields { + self.parse_tabular_array(length, &fields, depth, context) + } else { + // Non-tabular arrays as first field of list items require depth adjustment + // (items at depth +2 relative to hyphen, not the usual +1) + let adjusted_depth = match context { + ArrayParseContext::Normal => depth, + ArrayParseContext::ListItemFirstField => depth + 1, + }; + self.parse_regular_array(length, adjusted_depth) + } + })(); + + if pushed { + self.pop_delimiter(); } + + result } fn parse_tabular_array( @@ -818,8 +999,8 @@ impl<'a> Parser<'a> { // If something at the same indent level, it might be a new row (error) // unless it's a key-value pair (which belongs to parent) if actual_indent == expected_indent && !matches!(self.current_token, Token::Eof) { - let is_key_value = matches!(self.current_token, Token::String(_, _)) - && matches!(self.scanner.peek(), Some(':')); + let is_key_value = + self.is_key_token() && matches!(self.scanner.peek(), Some(':')); if !is_key_value { return Err(self.parse_error_with_context(format!( @@ -874,8 +1055,17 @@ impl<'a> Parser<'a> { Value::Object(Map::new()) } else if matches!(self.current_token, Token::LeftBracket) { self.parse_array(depth + 1)? - } else if let Token::String(s, _) = &self.current_token { - let key = s.clone(); + } else if self.is_key_token() { + let (key, key_was_quoted) = match self.key_from_token() { + Some(key) => key, + None => { + return Err(self.parse_error_with_context(format!( + "Expected key, found {:?}", + self.current_token + ))); + } + }; + self.validate_unquoted_key(&key, key_was_quoted)?; self.advance()?; if matches!(self.current_token, Token::Colon | Token::LeftBracket) { @@ -921,7 +1111,7 @@ impl<'a> Parser<'a> { } true } - } else if matches!(self.current_token, Token::String(_, _)) { + } else if self.is_key_token() { // When already positioned at a field key, check its indent let current_indent = self.scanner.get_last_line_indent(); current_indent == field_indent @@ -947,10 +1137,12 @@ impl<'a> Parser<'a> { break; } - let field_key = match &self.current_token { - Token::String(s, _) => s.clone(), - _ => break, - }; + let (field_key, field_key_was_quoted) = + match self.key_from_token() { + Some(key) => key, + None => break, + }; + self.validate_unquoted_key(&field_key, field_key_was_quoted)?; self.advance()?; let field_value = @@ -1109,12 +1301,14 @@ impl<'a> Parser<'a> { .into()) } } - Token::String(s, _) => { + Token::String(s, was_quoted) => { // Tabular fields can have multiple string tokens joined with spaces + self.validate_unquoted_string(s, *was_quoted)?; let mut accumulated = s.clone(); self.advance()?; - while let Token::String(next, _) = &self.current_token { + while let Token::String(next, next_was_quoted) = &self.current_token { + self.validate_unquoted_string(next, *next_was_quoted)?; if !accumulated.is_empty() { accumulated.push(' '); } @@ -1159,7 +1353,8 @@ impl<'a> Parser<'a> { .into()) } } - Token::String(s, _) => { + Token::String(s, was_quoted) => { + self.validate_unquoted_string(s, *was_quoted)?; let val = s.clone(); self.advance()?; Ok(Value::String(val)) diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index 9ca7615..bf71f13 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -27,6 +27,7 @@ pub struct Scanner { column: usize, active_delimiter: Option, last_line_indent: usize, + coerce_types: bool, } impl Scanner { @@ -39,6 +40,7 @@ impl Scanner { column: 1, active_delimiter: None, last_line_indent: 0, + coerce_types: true, } } @@ -47,6 +49,10 @@ impl Scanner { self.active_delimiter = delimiter; } + pub fn set_coerce_types(&mut self, coerce_types: bool) { + self.coerce_types = coerce_types; + } + /// Get the current position (line, column). pub fn current_position(&self) -> (usize, usize) { (self.line, self.column) @@ -293,6 +299,10 @@ impl Scanner { value.trim_end().to_string() }; + if !self.coerce_types { + return Ok(Token::String(value, false)); + } + match value.as_str() { "null" => Ok(Token::Null), "true" => Ok(Token::Bool(true)), @@ -326,6 +336,10 @@ impl Scanner { } fn parse_number(&self, s: &str) -> ToonResult { + if !self.coerce_types { + return Ok(Token::String(s.to_string(), false)); + } + // Number followed immediately by other chars like "0(f)" should be a string if let Some(next_ch) = self.peek() { if next_ch != ' ' @@ -354,6 +368,13 @@ impl Scanner { } } } + if s.starts_with("-0") && s.len() > 2 { + if let Some(third_char) = s.chars().nth(2) { + if third_char.is_ascii_digit() { + return Ok(Token::String(s.to_string(), false)); + } + } + } if s.contains('.') || s.contains('e') || s.contains('E') { if let Ok(f) = s.parse::() { @@ -369,11 +390,14 @@ impl Scanner { } /// Read the rest of the current line (until newline or EOF). - /// Returns the content with a flag indicating if it started with - /// whitespace. - pub fn read_rest_of_line_with_space_info(&mut self) -> (String, bool) { - let had_leading_space = matches!(self.peek(), Some(' ')); - self.skip_whitespace(); + /// Returns the content and any leading spaces between the current token + /// and the rest of the line. + pub fn read_rest_of_line_with_space_info(&mut self) -> (String, String) { + let mut leading_space = String::new(); + while matches!(self.peek(), Some(' ')) { + leading_space.push(' '); + self.advance(); + } let mut result = String::new(); while let Some(ch) = self.peek() { @@ -384,7 +408,7 @@ impl Scanner { self.advance(); } - (result.trim_end().to_string(), had_leading_space) + (result.trim_end().to_string(), leading_space) } /// Read the rest of the current line (until newline or EOF). @@ -448,6 +472,10 @@ impl Scanner { )); } + if !self.coerce_types { + return Ok(Token::String(trimmed.to_string(), false)); + } + match trimmed { "true" => return Ok(Token::Bool(true)), "false" => return Ok(Token::Bool(false)), @@ -455,13 +483,8 @@ impl Scanner { _ => {} } - if trimmed.starts_with('-') - || trimmed - .chars() - .next() - .is_some_and(|c| c.is_ascii_digit()) - { - // Leading zeros like "05" are strings + if trimmed.starts_with('-') || trimmed.chars().next().is_some_and(|c| c.is_ascii_digit()) { + // Leading zeros like "05" or "-05" are strings if trimmed.starts_with('0') && trimmed.len() > 1 { if let Some(second_char) = trimmed.chars().nth(1) { if second_char.is_ascii_digit() { @@ -469,6 +492,13 @@ impl Scanner { } } } + if trimmed.starts_with("-0") && trimmed.len() > 2 { + if let Some(third_char) = trimmed.chars().nth(2) { + if third_char.is_ascii_digit() { + return Ok(Token::String(trimmed.to_string(), false)); + } + } + } if trimmed.contains('.') || trimmed.contains('e') || trimmed.contains('E') { if let Ok(f) = trimmed.parse::() { @@ -595,24 +625,24 @@ mod tests { #[test] fn test_read_rest_of_line_with_space_info() { let mut scanner = Scanner::new(" world"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "world"); - assert!(had_space); + assert_eq!(leading_space, " "); let mut scanner = Scanner::new("world"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "world"); - assert!(!had_space); + assert!(leading_space.is_empty()); let mut scanner = Scanner::new("(hello)"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "(hello)"); - assert!(!had_space); + assert!(leading_space.is_empty()); let mut scanner = Scanner::new(""); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, leading_space) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, ""); - assert!(!had_space); + assert!(leading_space.is_empty()); } #[test] diff --git a/src/types/folding.rs b/src/types/folding.rs index 2c7d272..ce968f1 100644 --- a/src/types/folding.rs +++ b/src/types/folding.rs @@ -31,12 +31,12 @@ pub fn is_identifier_segment(s: &str) -> bool { None => return false, }; - if !first.is_alphabetic() && first != '_' { + if !first.is_ascii_alphabetic() && first != '_' { return false; } // Remaining characters: letters, digits, or underscore (NO dots) - chars.all(|c| c.is_alphanumeric() || c == '_') + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') } #[cfg(test)] diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 1b300cb..f8ad2f1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,6 +6,7 @@ pub mod validation; use indexmap::IndexMap; pub use literal::{is_keyword, is_literal_like, is_numeric_like, is_structural_char}; pub use number::format_canonical_number; +pub(crate) use string::is_valid_unquoted_key_decode; pub use string::{ escape_string, is_valid_unquoted_key, needs_quoting, quote_string, unescape_string, }; diff --git a/src/utils/string.rs b/src/utils/string.rs index 3622990..fc0b2a9 100644 --- a/src/utils/string.rs +++ b/src/utils/string.rs @@ -89,9 +89,7 @@ pub fn unescape_string(s: &str) -> Result { Ok(result) } -/// Check if a key can be written without quotes (alphanumeric, underscore, -/// dot). -pub fn is_valid_unquoted_key(key: &str) -> bool { +fn is_valid_unquoted_key_internal(key: &str, allow_hyphen: bool) -> bool { if key.is_empty() { return false; } @@ -103,11 +101,21 @@ pub fn is_valid_unquoted_key(key: &str) -> bool { return false; }; - if !first.is_alphabetic() && first != '_' { + if !first.is_ascii_alphabetic() && first != '_' { return false; } - chars.all(|c| c.is_alphanumeric() || c == '_' || c == '.') + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.' || (allow_hyphen && c == '-')) +} + +/// Check if a key can be written without quotes (alphanumeric, underscore, +/// dot). +pub fn is_valid_unquoted_key(key: &str) -> bool { + is_valid_unquoted_key_internal(key, false) +} + +pub(crate) fn is_valid_unquoted_key_decode(key: &str) -> bool { + is_valid_unquoted_key_internal(key, true) } /// Determine if a string needs quoting based on content and delimiter. @@ -295,5 +303,6 @@ mod tests { assert!(is_valid_unquoted_key("key.")); assert!(!is_valid_unquoted_key("key[value]")); assert!(!is_valid_unquoted_key("key{value}")); + assert!(is_valid_unquoted_key_decode("key-value")); } } From bde58a82b261e8c7848204c172aa52303342bce9 Mon Sep 17 00:00:00 2001 From: Bruno Meilick Date: Wed, 7 Jan 2026 14:02:31 +0000 Subject: [PATCH 11/11] test: add decoder edge cases --- tests/delimiters.rs | 10 +++---- tests/errors.rs | 8 +++--- tests/spec_edge_cases.rs | 59 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 tests/spec_edge_cases.rs diff --git a/tests/delimiters.rs b/tests/delimiters.rs index 7e52a67..dd0ef38 100644 --- a/tests/delimiters.rs +++ b/tests/delimiters.rs @@ -45,7 +45,7 @@ fn test_non_active_delimiters_in_tabular_arrays() { // data Per TOON spec §11: "non-active delimiters MUST NOT cause splits" // Test 1: Pipe character in value when comma is active delimiter (default) - let data = r#"item-list[1]{a,b}: + let data = r#""item-list"[1]{a,b}: ":",| "#; let decoded: Value = decode_default(data).unwrap(); @@ -53,7 +53,7 @@ fn test_non_active_delimiters_in_tabular_arrays() { assert_eq!(decoded["item-list"][0]["b"], "|"); // Test 2: Both values quoted - let data = r#"item-list[1]{a,b}: + let data = r#""item-list"[1]{a,b}: ":","|" "#; let decoded: Value = decode_default(data).unwrap(); @@ -61,13 +61,13 @@ fn test_non_active_delimiters_in_tabular_arrays() { assert_eq!(decoded["item-list"][0]["b"], "|"); // Test 3: Tab character in value when comma is active - let data = "item-list[1]{a,b}:\n \":\",\t\n"; + let data = "\"item-list\"[1]{a,b}:\n \":\",\"\\t\"\n"; let decoded: Value = decode_default(data).unwrap(); assert_eq!(decoded["item-list"][0]["a"], ":"); assert_eq!(decoded["item-list"][0]["b"], "\t"); // Test 4: Comma in value when pipe is active delimiter - should quote the comma - let data = r#"item-list[1|]{a|b}: + let data = r#""item-list"[1|]{a|b}: ":"|"," "#; let decoded: Value = decode_default(data).unwrap(); @@ -97,7 +97,7 @@ fn test_non_active_delimiters_in_inline_arrays() { fn test_delimiter_mismatch_error() { // Per TOON spec §6: delimiter in brackets must match delimiter in braces // This should error: pipe in brackets, comma in braces - let data = r#"item-list[1|]{a,b}: + let data = r#""item-list"[1|]{a,b}: ":",| "#; let result: Result = decode_default(data); diff --git a/tests/errors.rs b/tests/errors.rs index a100332..45fa38b 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -271,12 +271,12 @@ fn test_no_coercion_preserves_strings() { assert_eq!(result["value"], json!("true")); let result = decode::("value: 123", &opts).unwrap(); - assert!(result["value"].is_number()); - assert_eq!(result["value"], json!(123)); + assert!(result["value"].is_string()); + assert_eq!(result["value"], json!("123")); let result = decode::("value: true", &opts).unwrap(); - assert!(result["value"].is_boolean()); - assert_eq!(result["value"], json!(true)); + assert!(result["value"].is_string()); + assert_eq!(result["value"], json!("true")); } #[test] diff --git a/tests/spec_edge_cases.rs b/tests/spec_edge_cases.rs new file mode 100644 index 0000000..9fb5a8a --- /dev/null +++ b/tests/spec_edge_cases.rs @@ -0,0 +1,59 @@ +//! Spec compliance edge cases + +use serde_json::json; +use toon_format::types::PathExpansionMode; +use toon_format::{decode, decode_default, DecodeOptions}; + +#[test] +fn test_keyword_keys_allowed() { + let input = "true: 1\nfalse: 2\nnull: 3"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"true": 1, "false": 2, "null": 3})); +} + +#[test] +fn test_nested_array_delimiter_scoping() { + let input = "outer[2|]: [2]: a,b | [2]: c,d"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"outer": [["a", "b"], ["c", "d"]]})); +} + +#[test] +fn test_quoted_dotted_field_not_expanded() { + let input = "rows[1]{\"a.b\"}:\n 1"; + let opts = DecodeOptions::new().with_expand_paths(PathExpansionMode::Safe); + let result: serde_json::Value = decode(input, &opts).unwrap(); + assert_eq!(result, json!({"rows": [{"a.b": 1}]})); +} + +#[test] +fn test_negative_leading_zero_string() { + let input = "val: -05"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"val": "-05"})); +} + +#[test] +fn test_unquoted_tab_rejected_in_strict() { + let input = "val: a\tb"; + let result: Result = decode_default(input); + assert!(result.is_err()); +} + +#[test] +fn test_multiple_spaces_preserved() { + let input = "msg: hello world"; + let result: serde_json::Value = decode_default(input).unwrap(); + assert_eq!(result, json!({"msg": "hello world"})); +} + +#[test] +fn test_coerce_types_toggle() { + let input = "value: 123\nflag: true\nnone: null"; + let opts = DecodeOptions::new().with_coerce_types(false); + let result: serde_json::Value = decode(input, &opts).unwrap(); + assert_eq!( + result, + json!({"value": "123", "flag": "true", "none": "null"}) + ); +}