From 7acf9737542b4fad9ec7b64f81f1a3966b499601 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 27 Feb 2026 17:07:32 +0000
Subject: [PATCH] fix(parser): handle heredoc pipe ordering and edge cases

- Lexer saves rest-of-line after heredoc delimiter and re-injects it via
  a VecDeque buffer, so `cat <<EOF | sort` correctly pipes heredoc content
- Pipe continuation (`cat <<EOF |\nsort`) also works via the same mechanism
- Quote-aware rest-of-line scanning handles `cat <<EOF; echo "two\nthree"`
  where double-quoted strings span physical lines
- read_continuation_into concatenates adjacent quoted/unquoted segments
  after single-quoted strings, enabling partial-quote delimiters like
  `<<'EOF'"2"` -> delimiter `EOF2`
- Multiple heredocs on one line work naturally via repeated rest-of-line
  save/reinject (e.g., `while cat <<E1 && cat <<E2; do ... done`)
- While/until condition stdout is now captured and emitted, fixing
  `while cat <<EOF; do ... done` producing no condition output

Enables all 6 previously-skipped heredoc-edge spec tests.

Closes #359
---
 crates/bashkit/src/interpreter/mod.rs         |  18 +++
 crates/bashkit/src/parser/lexer.rs            | 129 +++++++++++++++---
 crates/bashkit/src/parser/mod.rs              | 121 +++++++++-------
 .../spec_cases/bash/heredoc-edge.test.sh      |   6 -
 4 files changed, 204 insertions(+), 70 deletions(-)

diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs
index 07b9f951..b42e76b0 100644
--- a/crates/bashkit/src/interpreter/mod.rs
+++ b/crates/bashkit/src/interpreter/mod.rs
@@ -1462,9 +1462,18 @@ impl Interpreter {
             self.counters.tick_loop(&self.limits)?;
 
             // Check condition (no errexit - conditions are expected to fail)
+            let emit_before_cond = self.output_emit_count;
             let condition_result = self
                 .execute_condition_sequence(&while_cmd.condition)
                 .await?;
+            // Condition commands produce visible output (e.g., `while cat <<EOF; do ... done`)
+            self.maybe_emit_output(
+                &condition_result.stdout,
+                &condition_result.stderr,
+                emit_before_cond,
+            );
+            stdout.push_str(&condition_result.stdout);
+            stderr.push_str(&condition_result.stderr);
             if condition_result.exit_code != 0 {
                 break;
             }
@@ -1547,9 +1556,18 @@ impl Interpreter {
             self.counters.tick_loop(&self.limits)?;
 
             // Check condition (no errexit - conditions are expected to fail)
+            let emit_before_cond = self.output_emit_count;
             let condition_result = self
                 .execute_condition_sequence(&until_cmd.condition)
                 .await?;
+            // Condition commands produce visible output
+            self.maybe_emit_output(
+                &condition_result.stdout,
+                &condition_result.stderr,
+                emit_before_cond,
+            );
+            stdout.push_str(&condition_result.stdout);
+            stderr.push_str(&condition_result.stderr);
             if condition_result.exit_code == 0 {
                 break;
             }
diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs
index 5a466868..8a10a299 100644
--- a/crates/bashkit/src/parser/lexer.rs
+++ b/crates/bashkit/src/parser/lexer.rs
@@ -2,6 +2,8 @@
 //!
 //! Tokenizes input into a stream of tokens with source position tracking.
 
+use std::collections::VecDeque;
+
 use super::span::{Position, Span};
 use super::tokens::Token;
 
@@ -19,9 +21,9 @@ pub struct Lexer<'a> {
     /// Current position in the input
     position: Position,
     chars: std::iter::Peekable<std::str::Chars<'a>>,
-    /// Rest-of-line text captured during heredoc parsing.
-    /// In `cat <<EOF > file`, this holds ` > file`.
-    pub heredoc_rest_of_line: String,
+    /// Buffer for re-injected characters (e.g., rest-of-line after heredoc delimiter).
+    /// Consumed before `chars`.
+    reinject_buf: VecDeque<char>,
 }
 
 impl<'a> Lexer<'a> {
@@ -31,7 +33,7 @@ impl<'a> Lexer<'a> {
             input,
             position: Position::new(),
             chars: input.chars().peekable(),
-            heredoc_rest_of_line: String::new(),
+            reinject_buf: VecDeque::new(),
         }
     }
 
@@ -47,11 +49,19 @@ impl<'a> Lexer<'a> {
     }
 
     fn peek_char(&mut self) -> Option<char> {
-        self.chars.peek().copied()
+        if let Some(&ch) = self.reinject_buf.front() {
+            Some(ch)
+        } else {
+            self.chars.peek().copied()
+        }
     }
 
     fn advance(&mut self) -> Option<char> {
-        let ch = self.chars.next();
+        let ch = if !self.reinject_buf.is_empty() {
+            self.reinject_buf.pop_front()
+        } else {
+            self.chars.next()
+        };
         if let Some(c) = ch {
             self.position.advance(c);
         }
@@ -708,10 +718,67 @@ impl<'a> Lexer<'a> {
             self.advance();
         }
 
+        // If next char is another quote or word char, concatenate (e.g., 'EOF'"2" -> EOF2).
+        // Any quoting makes the whole token literal.
+        self.read_continuation_into(&mut content);
+
         // Single-quoted strings are literal - no variable expansion
         Some(Token::LiteralWord(content))
     }
 
+    /// After a closing quote, read any adjacent quoted or unquoted word chars
+    /// into `content`.  Handles concatenation like `'foo'"bar"baz` -> `foobarbaz`.
+    fn read_continuation_into(&mut self, content: &mut String) {
+        loop {
+            match self.peek_char() {
+                Some('\'') => {
+                    self.advance(); // opening '
+                    while let Some(ch) = self.peek_char() {
+                        if ch == '\'' {
+                            self.advance(); // closing '
+                            break;
+                        }
+                        content.push(ch);
+                        self.advance();
+                    }
+                }
+                Some('"') => {
+                    self.advance(); // opening "
+                    while let Some(ch) = self.peek_char() {
+                        if ch == '"' {
+                            self.advance(); // closing "
+                            break;
+                        }
+                        if ch == '\\' {
+                            self.advance();
+                            if let Some(next) = self.peek_char() {
+                                match next {
+                                    '"' | '\\' | '$' | '`' => {
+                                        content.push(next);
+                                        self.advance();
+                                    }
+                                    _ => {
+                                        content.push('\\');
+                                        content.push(next);
+                                        self.advance();
+                                    }
+                                }
+                                continue;
+                            }
+                        }
+                        content.push(ch);
+                        self.advance();
+                    }
+                }
+                Some(ch) if self.is_word_char(ch) => {
+                    content.push(ch);
+                    self.advance();
+                }
+                _ => break,
+            }
+        }
+    }
+
     /// Read ANSI-C quoted content ($'...').
     /// Opening $' already consumed. Returns the resolved string.
     fn read_dollar_single_quoted_content(&mut self) -> String {
@@ -1212,16 +1279,34 @@ impl<'a> Lexer<'a> {
         let mut content = String::new();
         let mut current_line = String::new();
 
-        // Collect the rest of the command line after the heredoc delimiter.
-        // In bash, `cat <<EOF > file` means `> file` is still part of
-        // the command and should be parsed for redirections.
-        self.heredoc_rest_of_line.clear();
+        // Save rest of current line (after the delimiter token on the command line).
+        // For `cat <<EOF | sort`, this captures ` | sort` so the parser can
+        // tokenize the pipe and subsequent command after the heredoc body.
+        //
+        // Quoted strings may span multiple lines (e.g., `cat <<EOF; echo "two\nthree"`),
+        // so we track quoting state and continue across newlines until quotes close.
+        let mut rest_of_line = String::new();
+        let mut in_double_quote = false;
+        let mut in_single_quote = false;
         while let Some(ch) = self.peek_char() {
             self.advance();
-            if ch == '\n' {
+            if ch == '\n' && !in_double_quote && !in_single_quote {
                 break;
             }
-            self.heredoc_rest_of_line.push(ch);
+            if ch == '"' && !in_single_quote {
+                in_double_quote = !in_double_quote;
+            } else if ch == '\'' && !in_double_quote {
+                in_single_quote = !in_single_quote;
+            } else if ch == '\\' && in_double_quote {
+                // Escaped char inside double quotes — skip the next char too
+                rest_of_line.push(ch);
+                if let Some(next) = self.peek_char() {
+                    rest_of_line.push(next);
+                    self.advance();
+                }
+                continue;
+            }
+            rest_of_line.push(ch);
         }
 
         // Read lines until we find the delimiter
@@ -1254,6 +1339,15 @@ impl<'a> Lexer<'a> {
             }
         }
 
+        // Re-inject saved rest-of-line so subsequent tokens (pipes, commands, etc.)
+        // are visible to the parser. Add a newline so the tokenizer sees the line break.
+        if !rest_of_line.is_empty() {
+            for ch in rest_of_line.chars() {
+                self.reinject_buf.push_back(ch);
+            }
+            self.reinject_buf.push_back('\n');
+        }
+
         content
     }
 }
@@ -1370,7 +1464,6 @@ mod tests {
         let mut lexer = Lexer::new("\nhello\nworld\nEOF");
         let content = lexer.read_heredoc("EOF");
         assert_eq!(content, "hello\nworld\n");
-        assert_eq!(lexer.heredoc_rest_of_line.trim(), "");
     }
 
     #[test]
@@ -1378,7 +1471,6 @@ mod tests {
         let mut lexer = Lexer::new("\ntest\nEOF");
         let content = lexer.read_heredoc("EOF");
         assert_eq!(content, "test\n");
-        assert_eq!(lexer.heredoc_rest_of_line.trim(), "");
     }
 
     #[test]
@@ -1394,18 +1486,23 @@ mod tests {
         // Now read heredoc content
         let content = lexer.read_heredoc("EOF");
         assert_eq!(content, "hello\nworld\n");
-        assert_eq!(lexer.heredoc_rest_of_line.trim(), "");
     }
 
     #[test]
     fn test_read_heredoc_with_redirect() {
+        // Rest-of-line (> file.txt) is re-injected into the lexer buffer
         let mut lexer = Lexer::new("cat <<EOF > file.txt\nhello\nEOF");
         assert_eq!(lexer.next_token(), Some(Token::Word("cat".to_string())));
         assert_eq!(lexer.next_token(), Some(Token::HereDoc));
         assert_eq!(lexer.next_token(), Some(Token::Word("EOF".to_string())));
         let content = lexer.read_heredoc("EOF");
         assert_eq!(content, "hello\n");
-        assert_eq!(lexer.heredoc_rest_of_line.trim(), "> file.txt");
+        // The redirect tokens are now available from the lexer
+        assert_eq!(lexer.next_token(), Some(Token::RedirectOut));
+        assert_eq!(
+            lexer.next_token(),
+            Some(Token::Word("file.txt".to_string()))
+        );
     }
 
     #[test]
diff --git a/crates/bashkit/src/parser/mod.rs b/crates/bashkit/src/parser/mod.rs
index ad87adfa..c7673b6f 100644
--- a/crates/bashkit/src/parser/mod.rs
+++ b/crates/bashkit/src/parser/mod.rs
@@ -466,7 +466,9 @@ impl<'a> Parser<'a> {
                         kind,
                         target,
                     });
-                    break; // heredoc consumes rest of input line
+                    // Rest-of-line tokens re-injected by lexer; break so callers
+                    // can see pipes/semicolons.
+                    break;
                 }
                 _ => break,
             }
@@ -1769,10 +1771,9 @@ impl<'a> Parser<'a> {
                     // Don't advance - let read_heredoc consume directly from lexer position
 
                     // Read the here document content (reads until delimiter line).
-                    // Also captures rest-of-line text (e.g. `> file` in
-                    // `cat <<EOF > file`) into lexer.heredoc_rest_of_line.
+                    // Rest-of-line tokens (redirects, pipes, etc.) are re-injected
+                    // into the lexer buffer by read_heredoc.
                     let content = self.lexer.read_heredoc(&delimiter);
-                    let rest_of_line = std::mem::take(&mut self.lexer.heredoc_rest_of_line);
 
                     // Strip leading tabs for <<-
                     let content = if strip_tabs {
@@ -1810,56 +1811,49 @@ impl<'a> Parser<'a> {
                         target,
                     });
 
-                    // Parse rest-of-line for additional redirects
-                    // (e.g. `> file` in `cat <<EOF > file`).
-                    // We parse tokens directly instead of using parse_simple_command
-                    // because that method returns None for redirect-only input
-                    // (no command word), dropping the redirects we need.
-                    if !rest_of_line.trim().is_empty() {
-                        let mut sub = Parser::new(&rest_of_line);
-                        loop {
-                            match &sub.current_token {
-                                Some(tokens::Token::RedirectOut) => {
-                                    sub.advance();
-                                    if let Ok(target) = sub.expect_word() {
-                                        redirects.push(Redirect {
-                                            fd: None,
-                                            kind: RedirectKind::Output,
-                                            target,
-                                        });
-                                    }
+                    // Advance the parser token so it picks up re-injected
+                    // rest-of-line tokens (|, ;, &&, > file) from the lexer.
+                    self.advance();
+
+                    // If re-injected tokens include redirects (e.g. `> file`
+                    // in `cat <<EOF > file`), consume them here before breaking
+                    // out to pipeline/list parsers.
+                    while let Some(tok) = &self.current_token {
+                        match tok {
+                            tokens::Token::RedirectOut => {
+                                self.advance();
+                                if let Ok(target) = self.expect_word() {
+                                    redirects.push(Redirect {
+                                        fd: None,
+                                        kind: RedirectKind::Output,
+                                        target,
+                                    });
                                 }
-                                Some(tokens::Token::RedirectAppend) => {
-                                    sub.advance();
-                                    if let Ok(target) = sub.expect_word() {
-                                        redirects.push(Redirect {
-                                            fd: None,
-                                            kind: RedirectKind::Append,
-                                            target,
-                                        });
-                                    }
+                            }
+                            tokens::Token::RedirectAppend => {
+                                self.advance();
+                                if let Ok(target) = self.expect_word() {
+                                    redirects.push(Redirect {
+                                        fd: None,
+                                        kind: RedirectKind::Append,
+                                        target,
+                                    });
                                 }
-                                Some(tokens::Token::RedirectFd(fd)) => {
-                                    let fd = *fd;
-                                    sub.advance();
-                                    if let Ok(target) = sub.expect_word() {
-                                        redirects.push(Redirect {
-                                            fd: Some(fd),
-                                            kind: RedirectKind::Output,
-                                            target,
-                                        });
-                                    }
+                            }
+                            tokens::Token::RedirectFd(fd) => {
+                                let fd = *fd;
+                                self.advance();
+                                if let Ok(target) = self.expect_word() {
+                                    redirects.push(Redirect {
+                                        fd: Some(fd),
+                                        kind: RedirectKind::Output,
+                                        target,
+                                    });
                                 }
-                                _ => break,
                             }
+                            _ => break,
                         }
                     }
-
-                    // Now advance past the heredoc body
-                    self.advance();
-
-                    // Heredoc body consumed subsequent lines from input.
-                    // Stop parsing this command - next tokens belong to new commands.
                     break;
                 }
                 Some(tokens::Token::ProcessSubIn) | Some(tokens::Token::ProcessSubOut) => {
@@ -2788,4 +2782,35 @@ mod tests {
 
         assert!(matches!(&script.commands[0], Command::List(_)));
     }
+
+    #[test]
+    fn test_heredoc_pipe() {
+        let parser = Parser::new("cat <<EOF | sort\nc\na\nb\nEOF\n");
+        let script = parser.parse().unwrap();
+        assert!(
+            matches!(&script.commands[0], Command::Pipeline(_)),
+            "heredoc with pipe should parse as Pipeline"
+        );
+    }
+
+    #[test]
+    fn test_heredoc_multiple_on_line() {
+        let input = "while cat <<E1 && cat <<E2; do cat <<E3; break; done\n1\nE1\n2\nE2\n3\nE3\n";
+        let parser = Parser::new(input);
+        let script = parser.parse().unwrap();
+        assert_eq!(script.commands.len(), 1);
+        if let Command::Compound(comp, _) = &script.commands[0] {
+            if let CompoundCommand::While(w) = comp {
+                assert!(
+                    !w.condition.is_empty(),
+                    "while condition should be non-empty"
+                );
+                assert!(!w.body.is_empty(), "while body should be non-empty");
+            } else {
+                panic!("expected While compound command");
+            }
+        } else {
+            panic!("expected Compound command");
+        }
+    }
 }
diff --git a/crates/bashkit/tests/spec_cases/bash/heredoc-edge.test.sh b/crates/bashkit/tests/spec_cases/bash/heredoc-edge.test.sh
index 19868e16..46499c04 100644
--- a/crates/bashkit/tests/spec_cases/bash/heredoc-edge.test.sh
+++ b/crates/bashkit/tests/spec_cases/bash/heredoc-edge.test.sh
@@ -28,7 +28,6 @@ $var $(echo nope) $((1+2))
 
 ### heredoc_partial_quote_delimiter
 # Partial quote in delimiter still prevents expansion
-### skip: TODO partial quoting in heredoc delimiter not implemented
 cat <<'EOF'"2"
 one
 two
@@ -40,7 +39,6 @@ two
 
 ### heredoc_pipe_first_line
 # Here doc with pipe on first line
-### skip: TODO heredoc piped to sort - pipe ordering issue
 cat <<EOF | sort
 c
 a
@@ -54,7 +52,6 @@ c
 
 ### heredoc_pipe_last_line
 # Here doc with pipe continued on last line
-### skip: TODO heredoc pipe continuation - pipe ordering issue
 cat <<EOF |
 c
 a
@@ -94,7 +91,6 @@ X 3
 
 ### heredoc_in_while_condition
 # Here doc in while condition and body
-### skip: TODO multiple heredocs in while condition not parsed
 while cat <<E1 && cat <<E2; do cat <<E3; break; done
 1
 E1
@@ -110,7 +106,6 @@ E3
 
 ### heredoc_multiline_condition
 # Here doc in while condition on multiple lines
-### skip: TODO multiple heredocs in while condition not parsed
 while cat <<E1 && cat <<E2
 1
 E1
@@ -130,7 +125,6 @@ done
 
 ### heredoc_with_multiline_dquote
 # Here doc with multiline double quoted string
-### skip: TODO heredoc followed by multiline dquote on same line not parsed correctly
 cat <<EOF; echo "two
 three"
 one