From 3035ea1b35cea567f9ddbf50cd029c512238dba5 Mon Sep 17 00:00:00 2001 From: Matthaus Woolard Date: Sat, 6 Aug 2022 12:53:47 +1200 Subject: [PATCH 1/3] Add multi line text interpolation support --- Sources/Splash/Grammar/SwiftGrammar.swift | 91 +++++++++++++++++++++- Tests/SplashTests/Tests/LiteralTests.swift | 72 +++++++++++++++++ 2 files changed, 162 insertions(+), 1 deletion(-) diff --git a/Sources/Splash/Grammar/SwiftGrammar.swift b/Sources/Splash/Grammar/SwiftGrammar.swift index 79a98d5..360893e 100644 --- a/Sources/Splash/Grammar/SwiftGrammar.swift +++ b/Sources/Splash/Grammar/SwiftGrammar.swift @@ -189,7 +189,7 @@ private extension SwiftGrammar { return false } - return !segment.isWithinStringInterpolation + return !segment.isWithinMultiLineStringInterpolation } } @@ -605,6 +605,95 @@ private extension Segment { return markerCounts.start != markerCounts.end } + + var isWithinMultiLineStringInterpolation: Bool { + let delimiter = "\\(" + + if tokens.current == delimiter || tokens.previous == delimiter { + return true + } + + + /* + Loop back through tokens (not just same line) + counting closing ) and opening ( and to see if a \\( + can be found before the start of the string. + + if the number of closed braces is < the number of opening braces + 1 + then we are inside a multi line string interpolation. + */ + + var unbalancedClosedParenthesis = 0 + + /* Note the order of `(` and `)` matters. + + for example \( + this is an interpolation + )(but non of this is) + */ + + for token in tokens.all.lazy.reversed() { + var previousChar: Character? = nil + // only need to search to the start of this multi line string. + // multi line string must have new line after """ so will always be a suffix of a token. + if token.hasSuffix("\"\"\"") { + // We are before the first interpolation. + return false + } + + // The order of ( and ) is important> + // () does note close the interpolation + // )( does close the interpolation + for char in token.lazy.reversed() { + // we consume unbalancedClosedParenthesis + // only once we are sure we are not dealing with the start of + // an interpolation + if previousChar == "(" { + if char != "\\" { + if unbalancedClosedParenthesis > 0 { + unbalancedClosedParenthesis -= 1 + } + // we do not want to put unbalancedClosedParenthesis + // into negative as the order of ( and ) is very important. + } else { + // keeping ( in the previousChar + // so that if the token is \\( it still ends up consuming the open brane. + continue + } + } + + previousChar = char + + switch char { + case ")": + unbalancedClosedParenthesis += 1 + default: + previousChar = char + } + } + + + + if token.hasPrefix(delimiter) { + // there is a closing parenthesis that closes the scope + if unbalancedClosedParenthesis > 0 { + return false + } + // all the closing parenthesis have matching opening parenthesis. + return true + } + + // If the last char in the token was ( + if previousChar == "(" { + if unbalancedClosedParenthesis > 0 { + unbalancedClosedParenthesis -= 1 + } + } + } + + // not inside a multi line string + return false + } var isWithinStringInterpolation: Bool { let delimiter = "\\(" diff --git a/Tests/SplashTests/Tests/LiteralTests.swift b/Tests/SplashTests/Tests/LiteralTests.swift index 44c1575..d08a632 100644 --- a/Tests/SplashTests/Tests/LiteralTests.swift +++ b/Tests/SplashTests/Tests/LiteralTests.swift @@ -168,6 +168,78 @@ final class LiteralTests: SyntaxHighlighterTestCase { .token("\"\"\"", .string) ]) } + + func testMultiLineStringLiteralWithMultiLineInterpolated() { + let components = highlighter.highlight(""" + let string = \"\"\" + Hello\\( + variable, + format: .value + )(not-interpolated) + \"\"\" + """) + + XCTAssertEqual(components, [ + .token("let", .keyword), + .whitespace(" "), + .plainText("string"), + .whitespace(" "), + .plainText("="), + .whitespace(" "), + .token("\"\"\"", .string), + .whitespace("\n"), + .token("Hello", .string), + .plainText("\\("), + .whitespace("\n "), + .plainText("variable,"), + .whitespace("\n "), + .plainText("format:"), + .whitespace(" "), + .plainText("."), + .token("value", Splash.TokenType.dotAccess), + .whitespace("\n"), + .plainText(")"), + .token("(not-interpolated)", .string), + .whitespace("\n"), + .token("\"\"\"", .string) + ]) + } + + func testMultiLineStringLiteralWithInterpolatedString() { + let components = highlighter.highlight(""" + let string = \"\"\" + Hello \\( + value ? "Bob" + ) Welcome. + \"\"\" + """) + + XCTAssertEqual(components, [ + .token("let", .keyword), + .whitespace(" "), + .plainText("string"), + .whitespace(" "), + .plainText("="), + .whitespace(" "), + .token("\"\"\"", .string), + .whitespace("\n"), + .token("Hello", .string), + .whitespace(" "), + .plainText("\\("), + .whitespace("\n "), + .plainText("value"), + .whitespace(" "), + .plainText("?"), + .whitespace(" "), + .token("\"Bob\"", .string), + .whitespace("\n"), + .plainText(")"), + .whitespace(" "), + .token("Welcome.", .string), + .whitespace("\n"), + .token("\"\"\"", .string) + ]) + } func testSingleLineRawStringLiteral() { let components = highlighter.highlight(""" From 38b1eeba1e33de0843401420f1487d05bc90e720 Mon Sep 17 00:00:00 2001 From: Matthaus Woolard Date: Mon, 20 Jan 2025 21:41:19 +1300 Subject: [PATCH 2/3] update to swfit 6 --- Package.swift | 2 +- Sources/Splash/Theming/Theme.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 981b6d4..6134ba3 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.4 +// swift-tools-version:6.0 /** * Splash diff --git a/Sources/Splash/Theming/Theme.swift b/Sources/Splash/Theming/Theme.swift index 080014a..516759e 100644 --- a/Sources/Splash/Theming/Theme.swift +++ b/Sources/Splash/Theming/Theme.swift @@ -7,7 +7,7 @@ import Foundation #if !os(Linux) - +import AppKit /// A theme describes what fonts and colors to use when rendering /// certain output formats - such as `NSAttributedString`. Several /// default implementations are provided - see Theme+Defaults.swift. From 08d8b8b74f7b90449535b009f0a1672ecabe1a42 Mon Sep 17 00:00:00 2001 From: Matthaus Woolard Date: Mon, 20 Jan 2025 22:05:01 +1300 Subject: [PATCH 3/3] Remove recusive nature of .next() --- Sources/Splash/Tokenizing/Tokenizer.swift | 37 +++++++++++++++++----- Tests/SplashTests/Tests/LiteralTests.swift | 23 ++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/Sources/Splash/Tokenizing/Tokenizer.swift b/Sources/Splash/Tokenizing/Tokenizer.swift index 2431504..2f35e1e 100644 --- a/Sources/Splash/Tokenizing/Tokenizer.swift +++ b/Sources/Splash/Tokenizing/Tokenizer.swift @@ -58,14 +58,31 @@ private extension Tokenizer { self.grammar = grammar segments = (nil, nil) } - + mutating func next() -> Segment? { + while true { + switch self._next() { + case .next: + continue + case .segment(let segment): + return segment + } + } + } + + private enum NextResult { + case segment(Segment?) + case next + } + + private mutating func _next() -> NextResult { + let nextIndex = makeNextIndex() guard nextIndex != code.endIndex else { let segment = segments.current segments.current = nil - return segment + return .segment(segment) } index = nextIndex @@ -75,12 +92,14 @@ private extension Tokenizer { case .token, .delimiter: guard var segment = segments.current else { segments.current = makeSegment(with: component, at: nextIndex) - return next() + return .next } guard segment.trailingWhitespace == nil, component.isDelimiter == segment.currentTokenIsDelimiter else { - return finish(segment, with: component, at: nextIndex) + return .segment( + finish(segment, with: component, at: nextIndex) + ) } if component.isDelimiter { @@ -89,20 +108,22 @@ private extension Tokenizer { mergableWith: component.character) guard shouldMerge else { - return finish(segment, with: component, at: nextIndex) + return .segment( + finish(segment, with: component, at: nextIndex) + ) } } segment.tokens.current.append(component.character) segments.current = segment - return next() + return .next case .whitespace, .newline: guard var segment = segments.current else { var segment = makeSegment(with: component, at: nextIndex) segment.trailingWhitespace = component.token segment.isLastOnLine = component.isNewline segments.current = segment - return next() + return .next } if var existingWhitespace = segment.trailingWhitespace { @@ -117,7 +138,7 @@ private extension Tokenizer { } segments.current = segment - return next() + return .next } } diff --git a/Tests/SplashTests/Tests/LiteralTests.swift b/Tests/SplashTests/Tests/LiteralTests.swift index d08a632..11cd67c 100644 --- a/Tests/SplashTests/Tests/LiteralTests.swift +++ b/Tests/SplashTests/Tests/LiteralTests.swift @@ -382,4 +382,27 @@ final class LiteralTests: SyntaxHighlighterTestCase { .plainText(")") ]) } + + /** + + This test was adding since we ended up having a recursive loop that would crash. + + Switching to async mode reduces your accessible stack size. + */ + func testNestedMultiline() async { + let testString = #""" + struct ContentView: View { + @State private var text = """ + ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~ + """ + + var body: some View { + TextEditor(text: $text) + .font(.body) + .frame(width: 300, height: 300) + } + } + """# + let _ = highlighter.highlight(testString) + } }