From 505179e987e1bf0306133ea6549dfc5180d66ab3 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Sun, 14 Feb 2021 23:30:31 +0100 Subject: [PATCH 1/9] Removed unneeded escaping backslashes in regexes --- CommonRegexes.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CommonRegexes.h b/CommonRegexes.h index 1ff34804..c3df5f3b 100644 --- a/CommonRegexes.h +++ b/CommonRegexes.h @@ -8,17 +8,17 @@ namespace commonItems // catchall: // We grab everything that's NOT =, { or }, OR we grab everything within quotes, except newlines, which we already drop // in the parser. -static inline const char* catchallRegex = R"([^=^{^}]+|\".+\")"; +static inline const char* catchallRegex = R"([^=^{^}]+|".+")"; // numbers static inline const char* integerRegex = R"(-?\d+)"; -static inline const char* quotedIntegerRegex = R"(\"-?\d+\")"; +static inline const char* quotedIntegerRegex = R"("-?\d+")"; static inline const char* floatRegex = R"(-?\d+(.\d+)?)"; -static inline const char* quotedFloatRegex = R"(\"-?\d+(.\d+)?\")"; +static inline const char* quotedFloatRegex = R"("-?\d+(.\d+)?")"; // strings static inline const char* stringRegex = R"([^[:s:]^=^\{^\}^\"]+)"; -static inline const char* quotedStringRegex = R"(\"[^\n^=^\{^\}^\"]+\")"; +static inline const char* quotedStringRegex = R"("[^\n^=^\{^\}^\"]+")"; // compile time regexes, cool stuff From 8627f073a150e6b0c35ef701b88aec4c00879803 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Mon, 15 Feb 2021 00:50:43 +0100 Subject: [PATCH 2/9] registeredAnything class with derived classes --- Parser.cpp | 68 ++++++---------------- Parser.h | 78 ++++++++++++++++++++++++-- tests/CommonItemsTests.vcxproj.filters | 6 +- 3 files changed, 90 insertions(+), 62 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 6d0480db..b8bf0af1 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -39,13 +39,13 @@ void commonItems::parser::registerKeyword(const std::string& keyword, const pars void commonItems::parser::registerMatcher(bool (*matcher)(std::string_view), const parsingFunction& function) { - registeredMatchers.emplace_back(matcher, function); + registeredThings.push_back(std::make_unique(matcher, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function) { - generatedRegexes.emplace_back(std::regex(keyword), function); + registeredThings.push_back(std::make_unique(keyword, function)); } @@ -123,8 +123,7 @@ void commonItems::parser::clearRegisteredKeywords() noexcept { std::map().swap(registeredKeywordStrings); std::map().swap(registeredKeywordStringsStreamOnly); - std::vector>().swap(registeredMatchers); - std::vector>().swap(generatedRegexes); + std::vector>().swap(registeredThings); } @@ -146,33 +145,27 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt auto matched = tryToMatchAgainstKeywords(toReturn, strippedLexeme, isLexemeQuoted, theStream); - if (!matched) + // regexes and matchers + for (const auto& registered: registeredThings) { - for (const auto& [matcher, parsingFunction]: registeredMatchers) + if (registered->match(toReturn, theStream)) { - if (matcher(toReturn)) - { - parsingFunction(toReturn, theStream); - matched = true; - break; - } + matched = true; + break; } - if (!matched && isLexemeQuoted) + } + if (!matched && isLexemeQuoted) + { + for (const auto& registered: registeredThings) { - for (const auto& [matcher, parsingFunction]: registeredMatchers) + if (registered->matchStripped(toReturn, strippedLexeme, theStream)) { - if (matcher(strippedLexeme)) - { - parsingFunction(toReturn, theStream); - matched = true; - break; - } + matched = true; + break; } } } - if (!matched) - matched = tryToMatchAgainstRegexes(toReturn, strippedLexeme, isLexemeQuoted, theStream); - + if (!matched) gotToken = true; } @@ -217,35 +210,6 @@ inline bool commonItems::parser::tryToMatchAgainstKeywords(const std::string& to return false; } -inline bool commonItems::parser::tryToMatchAgainstRegexes(const std::string& toReturn, - const std::string& strippedLexeme, - bool isLexemeQuoted, - std::istream& theStream) -{ - for (const auto& [regex, parsingFunction]: generatedRegexes) - { - std::smatch match; - if (std::regex_match(toReturn, match, regex)) - { - parsingFunction(toReturn, theStream); - return true; - } - } - if (isLexemeQuoted) - { - for (const auto& [regex, parsingFunction]: generatedRegexes) - { - std::smatch match; - if (std::regex_match(strippedLexeme, match, regex)) - { - parsingFunction(toReturn, theStream); - return true; - } - } - } - return false; -} - std::optional commonItems::parser::getNextTokenWithoutMatching(std::istream& theStream) { diff --git a/Parser.h b/Parser.h index 783f264b..b54d3b1c 100644 --- a/Parser.h +++ b/Parser.h @@ -20,6 +20,76 @@ typedef std::function parsingFunctionStreamOnly; void absorbBOM(std::istream& theStream); +class registeredAnything +{ + public: + virtual ~registeredAnything() = default; + virtual bool match(const std::string& lexeme, std::istream& theStream) = 0; + virtual bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) = 0; +}; +class registeredRegex: public registeredAnything +{ + private: + std::regex regex; + parsingFunction function; + public: + registeredRegex(const std::string& keyword, const parsingFunction& function): + regex(std::regex(keyword)), function{function} + { + } + bool match(const std::string& lexeme, std::istream& theStream) + { + if (!std::regex_match(lexeme, regex)) + return false; + else + { + function(lexeme, theStream); + return true; + } + } + bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { + if (!std::regex_match(strippedLexeme, regex)) + return false; + else + { + function(lexeme, theStream); + return true; + } + } +}; +class registeredMatcher: public registeredAnything +{ + private: + bool (*matcher)(std::string_view); + parsingFunction function; + public: + registeredMatcher(bool (*matcher)(std::string_view), const parsingFunction& function): + matcher(matcher), function{function} + { + } + bool match(const std::string& lexeme, std::istream& theStream) + { + if (!matcher(lexeme)) + return false; + else + { + function(lexeme, theStream); + return true; + } + } + bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) + { + if (!matcher(strippedLexeme)) + return false; + else + { + function(lexeme, theStream); + return true; + } + } +}; + + class parser { public: @@ -50,15 +120,11 @@ class parser const std::string& strippedLexeme, bool isLexemeQuoted, std::istream& theStream); - inline bool tryToMatchAgainstRegexes(const std::string& toReturn, - const std::string& strippedLexeme, - bool isLexemeQuoted, - std::istream& theStream); std::map registeredKeywordStringsStreamOnly; std::map registeredKeywordStrings; - std::vector> registeredMatchers; - std::vector> generatedRegexes; + + std::vector> registeredThings; }; } // namespace commonItems diff --git a/tests/CommonItemsTests.vcxproj.filters b/tests/CommonItemsTests.vcxproj.filters index bfca185c..3280ba54 100644 --- a/tests/CommonItemsTests.vcxproj.filters +++ b/tests/CommonItemsTests.vcxproj.filters @@ -142,10 +142,8 @@ TestFiles - - - + TestFiles - + \ No newline at end of file From f18f77f4107f0fdd61bbed32915c83873b3023b6 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Mon, 15 Feb 2021 01:52:02 +0100 Subject: [PATCH 3/9] StreamOnly versions of registerRegex and registerMatcher --- Parser.cpp | 156 ++++++++++++++++++++++++++++++++++++++++++++++++----- Parser.h | 80 ++++++++++++--------------- 2 files changed, 176 insertions(+), 60 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index b8bf0af1..2230c104 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -11,6 +11,121 @@ namespace fs = std::filesystem; namespace commonItems { std::string getNextLexeme(std::istream& theStream); + + +registeredMatcher::registeredMatcher(bool (*matcher)(std::string_view), const parsingFunction& function): matcher(matcher), function{function} {} + +bool registeredMatcher::match(const std::string& lexeme, std::istream& theStream) +{ + if (!matcher(lexeme)) + return false; + else + { + function(lexeme, theStream); + return true; + } +} + +bool registeredMatcher::matchStripped(const std::string& lexeme, + const std::string& strippedLexeme, + std::istream& theStream) +{ + if (!matcher(strippedLexeme)) + return false; + else + { + function(lexeme, theStream); + return true; + } +} + + +registeredMatcherStreamOnly::registeredMatcherStreamOnly(bool (*matcher)(std::string_view), + const parsingFunctionStreamOnly& function): + matcher(matcher), function{function} +{ +} + +bool registeredMatcherStreamOnly::match(const std::string& lexeme, std::istream& theStream) +{ + if (!matcher(lexeme)) + return false; + else + { + function(theStream); + return true; + } +} + +bool registeredMatcherStreamOnly::matchStripped(const std::string& lexeme, + const std::string& strippedLexeme, + std::istream& theStream) +{ + if (!matcher(strippedLexeme)) + return false; + else + { + function(theStream); + return true; + } +} + + +registeredRegex::registeredRegex(const std::string& keyword, const parsingFunction& function): regex(std::regex(keyword)), function{function} {} + +bool registeredRegex::match(const std::string& lexeme, std::istream& theStream) +{ + if (!std::regex_match(lexeme, regex)) + return false; + else + { + function(lexeme, theStream); + return true; + } +} + +bool registeredRegex::matchStripped(const std::string& lexeme, + const std::string& strippedLexeme, + std::istream& theStream) +{ + if (!std::regex_match(strippedLexeme, regex)) + return false; + else + { + function(lexeme, theStream); + return true; + } +} + + +registeredRegexStreamOnly::registeredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): + regex(std::regex(keyword)), function{function} +{ +} + +bool registeredRegexStreamOnly::match(const std::string& lexeme, std::istream& theStream) +{ + if (!std::regex_match(lexeme, regex)) + return false; + else + { + function(theStream); + return true; + } +} + +bool registeredRegexStreamOnly::matchStripped(const std::string& lexeme, + const std::string& strippedLexeme, + std::istream& theStream) +{ + if (!std::regex_match(strippedLexeme, regex)) + return false; + else + { + function(theStream); + return true; + } +} } // namespace commonItems @@ -39,13 +154,25 @@ void commonItems::parser::registerKeyword(const std::string& keyword, const pars void commonItems::parser::registerMatcher(bool (*matcher)(std::string_view), const parsingFunction& function) { - registeredThings.push_back(std::make_unique(matcher, function)); + registeredThings.emplace_back(std::make_unique(matcher, function)); +} + + +void commonItems::parser::registerMatcher(bool (*matcher)(std::string_view), const parsingFunctionStreamOnly& function) +{ + registeredThings.emplace_back(std::make_unique(matcher, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function) { - registeredThings.push_back(std::make_unique(keyword, function)); + registeredThings.emplace_back(std::make_unique(keyword, function)); +} + + +void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunctionStreamOnly& function) +{ + registeredThings.emplace_back(std::make_unique(keyword, function)); } @@ -146,24 +273,27 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt auto matched = tryToMatchAgainstKeywords(toReturn, strippedLexeme, isLexemeQuoted, theStream); // regexes and matchers - for (const auto& registered: registeredThings) - { - if (registered->match(toReturn, theStream)) - { - matched = true; - break; - } - } - if (!matched && isLexemeQuoted) + if (!matched) { - for (const auto& registered: registeredThings) + for (const auto& registered: registeredThings) { - if (registered->matchStripped(toReturn, strippedLexeme, theStream)) + if (registered->match(toReturn, theStream)) { matched = true; break; } } + if (!matched && isLexemeQuoted) + { + for (const auto& registered: registeredThings) + { + if (registered->matchStripped(toReturn, strippedLexeme, theStream)) + { + matched = true; + break; + } + } + } } if (!matched) diff --git a/Parser.h b/Parser.h index b54d3b1c..96cbe9dd 100644 --- a/Parser.h +++ b/Parser.h @@ -27,68 +27,52 @@ class registeredAnything virtual bool match(const std::string& lexeme, std::istream& theStream) = 0; virtual bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) = 0; }; + class registeredRegex: public registeredAnything { private: std::regex regex; parsingFunction function; public: - registeredRegex(const std::string& keyword, const parsingFunction& function): - regex(std::regex(keyword)), function{function} - { - } - bool match(const std::string& lexeme, std::istream& theStream) - { - if (!std::regex_match(lexeme, regex)) - return false; - else - { - function(lexeme, theStream); - return true; - } - } - bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { - if (!std::regex_match(strippedLexeme, regex)) - return false; - else - { - function(lexeme, theStream); - return true; - } - } + registeredRegex(const std::string& keyword, const parsingFunction& function); + bool match(const std::string& lexeme, std::istream& theStream); + bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); +}; + +class registeredRegexStreamOnly: public registeredAnything +{ + private: + std::regex regex; + parsingFunctionStreamOnly function; + public: + registeredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function); + bool match(const std::string& lexeme, std::istream& theStream); + bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; + class registeredMatcher: public registeredAnything { private: bool (*matcher)(std::string_view); parsingFunction function; public: - registeredMatcher(bool (*matcher)(std::string_view), const parsingFunction& function): - matcher(matcher), function{function} - { - } - bool match(const std::string& lexeme, std::istream& theStream) - { - if (!matcher(lexeme)) - return false; - else - { - function(lexeme, theStream); - return true; - } - } - bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) - { - if (!matcher(strippedLexeme)) - return false; - else - { - function(lexeme, theStream); - return true; - } - } + registeredMatcher(bool (*matcher)(std::string_view), const parsingFunction& function); + bool match(const std::string& lexeme, std::istream& theStream); + bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; +class registeredMatcherStreamOnly: public registeredAnything +{ + private: + bool (*matcher)(std::string_view); + parsingFunctionStreamOnly function; + public: + registeredMatcherStreamOnly(bool (*matcher)(std::string_view), const parsingFunctionStreamOnly& function); + bool match(const std::string& lexeme, std::istream& theStream); + bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); +}; + + class parser { @@ -103,7 +87,9 @@ class parser void registerKeyword(const std::string& keyword, const parsingFunctionStreamOnly& function); void registerKeyword(const std::string& keyword, const parsingFunction& function); // for the few keywords that need to be returned // for compile time regex matchers, but will work with any function that has the same return and argument type + void registerMatcher(bool (*matcher)(std::string_view), const parsingFunctionStreamOnly& function); void registerMatcher(bool (*matcher)(std::string_view), const parsingFunction& function); + void registerRegex(const std::string& keyword, const parsingFunctionStreamOnly& function); void registerRegex(const std::string& keyword, const parsingFunction& function); void clearRegisteredKeywords() noexcept; From 89bd81b08719f6acdbd867adac3a96ffad5a0479 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Mon, 15 Feb 2021 02:11:27 +0100 Subject: [PATCH 4/9] Purge of "unused" in ParserHelpers --- ParserHelpers.cpp | 8 ++++---- ParserHelpers.h | 6 +++--- tests/ParserHelperTests.cpp | 32 ++++++++++++++++---------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ParserHelpers.cpp b/ParserHelpers.cpp index 92b81374..498dfc69 100644 --- a/ParserHelpers.cpp +++ b/ParserHelpers.cpp @@ -13,7 +13,7 @@ namespace commonItems std::string getNextLexeme(std::istream& theStream); -void ignoreItem(const std::string& unused, std::istream& theStream) +void ignoreItem(std::istream& theStream) { auto next = getNextLexeme(theStream); if (next == "=") @@ -59,7 +59,7 @@ void ignoreItem(const std::string& unused, std::istream& theStream) } -void ignoreObject(const std::string& unused, std::istream& theStream) +void ignoreObject(std::istream& theStream) { auto braceDepth = 0; while (true) @@ -86,7 +86,7 @@ void ignoreObject(const std::string& unused, std::istream& theStream) } -void ignoreString(const std::string& unused, std::istream& theStream) +void ignoreString(std::istream& theStream) { singleString ignore(theStream); } @@ -510,7 +510,7 @@ stringsOfItems::stringsOfItems(std::istream& theStream) stringsOfItemNames::stringsOfItemNames(std::istream& theStream) { registerMatcher(catchallRegexMatch, [this](const std::string& itemName, std::istream& theStream) { - ignoreItem(itemName, theStream); + ignoreItem(theStream); theStrings.push_back(itemName); }); diff --git a/ParserHelpers.h b/ParserHelpers.h index 40300952..a5599153 100644 --- a/ParserHelpers.h +++ b/ParserHelpers.h @@ -11,9 +11,9 @@ namespace commonItems { -void ignoreItem(const std::string& unused, std::istream& theStream); -void ignoreObject(const std::string& unused, std::istream& theStream); -void ignoreString(const std::string& unused, std::istream& theStream); +void ignoreItem(std::istream& theStream); +void ignoreObject(std::istream& theStream); +void ignoreString(std::istream& theStream); template [[nodiscard]] std::enable_if_t, T> stringToInteger(const std::string& str, bool skipPartialMatchWarning = false); diff --git a/tests/ParserHelperTests.cpp b/tests/ParserHelperTests.cpp index fbdb0e1c..fba58e76 100644 --- a/tests/ParserHelperTests.cpp +++ b/tests/ParserHelperTests.cpp @@ -8,7 +8,7 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresSimpleText) { std::stringstream input{"ignore_me More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -20,7 +20,7 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresAssignedText) { std::stringstream input{"= ignore_me More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -32,7 +32,7 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresBracedItem) { std::stringstream input{"{ { ignore_me } } More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -44,7 +44,7 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresAssignedBracedItem) { std::stringstream input{"= { { ignore_me } } More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -56,7 +56,7 @@ TEST(ParserHelper_Tests, IgnoreObjectIgnoresNextItem) { std::stringstream input{"ignore_me More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -68,7 +68,7 @@ TEST(ParserHelper_Tests, IgnoreObjectIgnoresWholeBracedItem) { std::stringstream input{"{ { ignore_me } } More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -80,7 +80,7 @@ TEST(ParserHelper_Tests, IgnoreStringIgnoresNextItem) { std::stringstream input{"ignore_me More text"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -92,7 +92,7 @@ TEST(ParserHelper_Tests, IgnoreStringIgnoresWholeQuoation) { std::stringstream input{R"("ignore_me More" text)"}; input >> std::noskipws; - commonItems::ignoreItem("unused", input); + commonItems::ignoreItem(input); char buffer[256]; input.getline(buffer, sizeof buffer); @@ -898,8 +898,8 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresSimpleColorWithColorSpace) std::stringstream input2{"hsv {0.1 1.0 0.6} More text"}; input >> std::noskipws; input2 >> std::noskipws; - commonItems::ignoreItem("unused", input); - commonItems::ignoreItem("unused", input2); + commonItems::ignoreItem(input); + commonItems::ignoreItem(input2); char buffer[256]; char buffer2[256]; @@ -916,8 +916,8 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresAssignedColorWithColorSpace) std::stringstream input2{"= hsv {0.1 1.0 0.6} More text"}; input >> std::noskipws; input2 >> std::noskipws; - commonItems::ignoreItem("unused", input); - commonItems::ignoreItem("unused", input2); + commonItems::ignoreItem(input); + commonItems::ignoreItem(input2); char buffer[256]; char buffer2[256]; @@ -934,8 +934,8 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresRgbAndHsvStringsWithoutBreakingParsing std::stringstream input2{"= hsv next_parameter = 420 More text"}; input >> std::noskipws; input2 >> std::noskipws; - commonItems::ignoreItem("unused", input); - commonItems::ignoreItem("unused", input2); + commonItems::ignoreItem(input); + commonItems::ignoreItem(input2); char buffer[256]; char buffer2[256]; @@ -952,8 +952,8 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresQuotedRgbAndHsvStringsWithoutBreakingP std::stringstream input2{"= \"hsv\" next_parameter = 420 More text"}; input >> std::noskipws; input2 >> std::noskipws; - commonItems::ignoreItem("unused", input); - commonItems::ignoreItem("unused", input2); + commonItems::ignoreItem(input); + commonItems::ignoreItem(input2); char buffer[256]; char buffer2[256]; From b8d0e06bba364fb435ae7a51af47ce3b9e720f46 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Mon, 15 Feb 2021 02:17:28 +0100 Subject: [PATCH 5/9] Renamed registeredThings to registeredRegexesAndMatchers --- Parser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser.h b/Parser.h index 96cbe9dd..87ad3f3f 100644 --- a/Parser.h +++ b/Parser.h @@ -110,7 +110,7 @@ class parser std::map registeredKeywordStringsStreamOnly; std::map registeredKeywordStrings; - std::vector> registeredThings; + std::vector> registeredRegexesAndMatchers; }; } // namespace commonItems From 536dbeaecc9b077b01792a6a08dd5f4bfc2e7212 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Mon, 15 Feb 2021 02:17:53 +0100 Subject: [PATCH 6/9] Update Parser.cpp --- Parser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 2230c104..29d1e5bf 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -275,7 +275,7 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt // regexes and matchers if (!matched) { - for (const auto& registered: registeredThings) + for (const auto& registered: registeredRegexesAndMatchers) { if (registered->match(toReturn, theStream)) { @@ -285,7 +285,7 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt } if (!matched && isLexemeQuoted) { - for (const auto& registered: registeredThings) + for (const auto& registered: registeredRegexesAndMatchers) { if (registered->matchStripped(toReturn, strippedLexeme, theStream)) { From a76506a2af80160b5e4c4dbfcce254cb6a0dbec2 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Wed, 24 Feb 2021 15:37:41 +0100 Subject: [PATCH 7/9] Update Parser.cpp --- Parser.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 29d1e5bf..a9e6f3f4 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -154,25 +154,25 @@ void commonItems::parser::registerKeyword(const std::string& keyword, const pars void commonItems::parser::registerMatcher(bool (*matcher)(std::string_view), const parsingFunction& function) { - registeredThings.emplace_back(std::make_unique(matcher, function)); + registeredRegexesAndMatchers.emplace_back(std::make_unique(matcher, function)); } void commonItems::parser::registerMatcher(bool (*matcher)(std::string_view), const parsingFunctionStreamOnly& function) { - registeredThings.emplace_back(std::make_unique(matcher, function)); + registeredRegexesAndMatchers.emplace_back(std::make_unique(matcher, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function) { - registeredThings.emplace_back(std::make_unique(keyword, function)); + registeredRegexesAndMatchers.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunctionStreamOnly& function) { - registeredThings.emplace_back(std::make_unique(keyword, function)); + registeredRegexesAndMatchers.emplace_back(std::make_unique(keyword, function)); } @@ -250,7 +250,7 @@ void commonItems::parser::clearRegisteredKeywords() noexcept { std::map().swap(registeredKeywordStrings); std::map().swap(registeredKeywordStringsStreamOnly); - std::vector>().swap(registeredThings); + std::vector>().swap(registeredRegexesAndMatchers); } From faf7f4036e77a3be7ba9bc8913d911d3c2b3692c Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Fri, 30 Apr 2021 15:43:45 +0200 Subject: [PATCH 8/9] keywords and regexes in one vector --- Parser.cpp | 127 +++++++++++++++------------------------------- Parser.h | 39 ++++++-------- ParserHelpers.cpp | 2 +- 3 files changed, 57 insertions(+), 111 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 437a75bb..5ef2a625 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -13,12 +13,14 @@ namespace commonItems std::string getNextLexeme(std::istream& theStream); -registeredMatcher::registeredMatcher(bool (*matcher)(std::string_view), const parsingFunction& function): matcher(matcher), function{function} {} +RegisteredKeyword::RegisteredKeyword(const std::string& keyword, const parsingFunction& function): keyword(keyword), function(function) {} -bool registeredMatcher::match(const std::string& lexeme, std::istream& theStream) +bool RegisteredKeyword::match(const std::string& lexeme, std::istream& theStream) { - if (!matcher(lexeme)) + if (lexeme != keyword) + { return false; + } else { function(lexeme, theStream); @@ -26,12 +28,12 @@ bool registeredMatcher::match(const std::string& lexeme, std::istream& theStream } } -bool registeredMatcher::matchStripped(const std::string& lexeme, - const std::string& strippedLexeme, - std::istream& theStream) +bool RegisteredKeyword::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { - if (!matcher(strippedLexeme)) + if (strippedLexeme != keyword) + { return false; + } else { function(lexeme, theStream); @@ -40,16 +42,14 @@ bool registeredMatcher::matchStripped(const std::string& lexeme, } -registeredMatcherStreamOnly::registeredMatcherStreamOnly(bool (*matcher)(std::string_view), - const parsingFunctionStreamOnly& function): - matcher(matcher), function{function} -{ -} +RegisteredKeywordStreamOnly::RegisteredKeywordStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): keyword(keyword), function(function) {} -bool registeredMatcherStreamOnly::match(const std::string& lexeme, std::istream& theStream) +bool RegisteredKeywordStreamOnly::match(const std::string& lexeme, std::istream& theStream) { - if (!matcher(lexeme)) + if (lexeme != keyword) + { return false; + } else { function(theStream); @@ -57,12 +57,12 @@ bool registeredMatcherStreamOnly::match(const std::string& lexeme, std::istream& } } -bool registeredMatcherStreamOnly::matchStripped(const std::string& lexeme, - const std::string& strippedLexeme, - std::istream& theStream) +bool RegisteredKeywordStreamOnly::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { - if (!matcher(strippedLexeme)) + if (strippedLexeme != keyword) + { return false; + } else { function(theStream); @@ -71,9 +71,9 @@ bool registeredMatcherStreamOnly::matchStripped(const std::string& lexeme, } -registeredRegex::registeredRegex(const std::string& keyword, const parsingFunction& function): regex(std::regex(keyword)), function{function} {} +RegisteredRegex::RegisteredRegex(const std::string& keyword, const parsingFunction& function): regex(std::regex(keyword)), function{function} {} -bool registeredRegex::match(const std::string& lexeme, std::istream& theStream) +bool RegisteredRegex::match(const std::string& lexeme, std::istream& theStream) { if (!std::regex_match(lexeme, regex)) return false; @@ -84,7 +84,7 @@ bool registeredRegex::match(const std::string& lexeme, std::istream& theStream) } } -bool registeredRegex::matchStripped(const std::string& lexeme, +bool RegisteredRegex::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { @@ -98,12 +98,9 @@ bool registeredRegex::matchStripped(const std::string& lexeme, } -registeredRegexStreamOnly::registeredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): - regex(std::regex(keyword)), function{function} -{ -} +RegisteredRegexStreamOnly::RegisteredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): regex(std::regex(keyword)), function{function} {} -bool registeredRegexStreamOnly::match(const std::string& lexeme, std::istream& theStream) +bool RegisteredRegexStreamOnly::match(const std::string& lexeme, std::istream& theStream) { if (!std::regex_match(lexeme, regex)) return false; @@ -114,7 +111,7 @@ bool registeredRegexStreamOnly::match(const std::string& lexeme, std::istream& t } } -bool registeredRegexStreamOnly::matchStripped(const std::string& lexeme, +bool RegisteredRegexStreamOnly::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { @@ -142,25 +139,25 @@ void commonItems::absorbBOM(std::istream& theStream) void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunction& function) { - registeredKeywordStrings.insert(std::make_pair(keyword, function)); + registered.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunctionStreamOnly& function) { - registeredKeywordStringsStreamOnly.insert(std::make_pair(keyword, function)); + registered.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function) { - registeredRegexesAndMatchers.emplace_back(std::make_unique(keyword, function)); + registered.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunctionStreamOnly& function) { - registeredRegexesAndMatchers.emplace_back(std::make_unique(keyword, function)); + registered.emplace_back(std::make_unique(keyword, function)); } @@ -236,9 +233,7 @@ void commonItems::parser::parseFile(std::string_view filename) void commonItems::parser::clearRegisteredKeywords() noexcept { - std::map().swap(registeredKeywordStrings); - std::map().swap(registeredKeywordStringsStreamOnly); - std::vector>().swap(registeredRegexesAndMatchers); + std::vector>().swap(registered); } @@ -258,28 +253,23 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt const auto strippedLexeme = remQuotes(toReturn); const auto isLexemeQuoted = (strippedLexeme.size() < toReturn.size()); - auto matched = tryToMatchAgainstKeywords(toReturn, strippedLexeme, isLexemeQuoted, theStream); - - // regexes and matchers - if (!matched) + bool matched = false; + for (const auto& registered: registered) { - for (const auto& registered: registeredRegexesAndMatchers) + if (registered->match(toReturn, theStream)) { - if (registered->match(toReturn, theStream)) - { - matched = true; - break; - } + matched = true; + break; } - if (!matched && isLexemeQuoted) + } + if (!matched && isLexemeQuoted) + { + for (const auto& registered: registered) { - for (const auto& registered: registeredRegexesAndMatchers) + if (registered->matchStripped(toReturn, strippedLexeme, theStream)) { - if (registered->matchStripped(toReturn, strippedLexeme, theStream)) - { - matched = true; - break; - } + matched = true; + break; } } } @@ -294,41 +284,6 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt } -inline bool commonItems::parser::tryToMatchAgainstKeywords(const std::string& toReturn, - const std::string& strippedLexeme, - bool isLexemeQuoted, - std::istream& theStream) -{ - if (const auto& match = registeredKeywordStringsStreamOnly.find(toReturn); match != registeredKeywordStringsStreamOnly.end()) - { - match->second(theStream); - return true; - } - else if (const auto& match = registeredKeywordStrings.find(toReturn); match != registeredKeywordStrings.end()) - { - match->second(toReturn, theStream); - return true; - } - else if (isLexemeQuoted) - { - if (const auto& strippedMatch = registeredKeywordStringsStreamOnly.find(strippedLexeme); - strippedMatch != registeredKeywordStringsStreamOnly.end()) - { - strippedMatch->second(theStream); - return true; - } - else if (const auto& strippedMatch = registeredKeywordStrings.find(strippedLexeme); - strippedMatch != registeredKeywordStrings.end()) - { - strippedMatch->second(toReturn, theStream); - return true; - } - } - - return false; -} - - std::optional commonItems::parser::getNextTokenWithoutMatching(std::istream& theStream) { theStream >> std::noskipws; diff --git a/Parser.h b/Parser.h index 51115974..7fd823ab 100644 --- a/Parser.h +++ b/Parser.h @@ -20,60 +20,59 @@ typedef std::function parsingFunctionStreamOnly; void absorbBOM(std::istream& theStream); -class registeredAnything +class Registered { public: - virtual ~registeredAnything() = default; + virtual ~Registered() = default; virtual bool match(const std::string& lexeme, std::istream& theStream) = 0; virtual bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) = 0; }; -class registeredRegex: public registeredAnything +class RegisteredKeyword: public Registered { private: - std::regex regex; + std::string keyword; parsingFunction function; public: - registeredRegex(const std::string& keyword, const parsingFunction& function); + RegisteredKeyword(const std::string& keyword, const parsingFunction& function); bool match(const std::string& lexeme, std::istream& theStream); bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; -class registeredRegexStreamOnly: public registeredAnything +class RegisteredKeywordStreamOnly: public Registered { private: - std::regex regex; + std::string keyword; parsingFunctionStreamOnly function; public: - registeredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function); + RegisteredKeywordStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function); bool match(const std::string& lexeme, std::istream& theStream); bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; -class registeredMatcher: public registeredAnything +class RegisteredRegex: public Registered { private: - bool (*matcher)(std::string_view); + std::regex regex; parsingFunction function; public: - registeredMatcher(bool (*matcher)(std::string_view), const parsingFunction& function); + RegisteredRegex(const std::string& keyword, const parsingFunction& function); bool match(const std::string& lexeme, std::istream& theStream); bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; -class registeredMatcherStreamOnly: public registeredAnything +class RegisteredRegexStreamOnly: public Registered { private: - bool (*matcher)(std::string_view); + std::regex regex; parsingFunctionStreamOnly function; public: - registeredMatcherStreamOnly(bool (*matcher)(std::string_view), const parsingFunctionStreamOnly& function); + RegisteredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function); bool match(const std::string& lexeme, std::istream& theStream); bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; - class parser { public: @@ -99,15 +98,7 @@ class parser private: - inline bool tryToMatchAgainstKeywords(const std::string& toReturn, - const std::string& strippedLexeme, - bool isLexemeQuoted, - std::istream& theStream); - - std::map registeredKeywordStringsStreamOnly; - std::map registeredKeywordStrings; - - std::vector> registeredRegexesAndMatchers; + std::vector> registered; }; } // namespace commonItems diff --git a/ParserHelpers.cpp b/ParserHelpers.cpp index cea7eefe..8d5c495f 100644 --- a/ParserHelpers.cpp +++ b/ParserHelpers.cpp @@ -514,7 +514,7 @@ stringsOfItems::stringsOfItems(std::istream& theStream) stringsOfItemNames::stringsOfItemNames(std::istream& theStream) { registerRegex(catchallRegex, [this](const std::string& itemName, std::istream& theStream) { - ignoreItem(itemName, theStream); + ignoreItem(theStream); theStrings.push_back(itemName); }); From ca0489f078fbe3737fa60610d26a5ddeb2c3e9c9 Mon Sep 17 00:00:00 2001 From: IlikeTrains Date: Fri, 30 Apr 2021 18:29:11 +0200 Subject: [PATCH 9/9] register keywords as regexes --- Parser.cpp | 83 ++++++++---------------------------------------------- Parser.h | 36 ++++++----------------- 2 files changed, 20 insertions(+), 99 deletions(-) diff --git a/Parser.cpp b/Parser.cpp index 5ef2a625..e3ebd72b 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -13,65 +13,10 @@ namespace commonItems std::string getNextLexeme(std::istream& theStream); -RegisteredKeyword::RegisteredKeyword(const std::string& keyword, const parsingFunction& function): keyword(keyword), function(function) {} - -bool RegisteredKeyword::match(const std::string& lexeme, std::istream& theStream) -{ - if (lexeme != keyword) - { - return false; - } - else - { - function(lexeme, theStream); - return true; - } -} - -bool RegisteredKeyword::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) -{ - if (strippedLexeme != keyword) - { - return false; - } - else - { - function(lexeme, theStream); - return true; - } -} - - -RegisteredKeywordStreamOnly::RegisteredKeywordStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): keyword(keyword), function(function) {} - -bool RegisteredKeywordStreamOnly::match(const std::string& lexeme, std::istream& theStream) -{ - if (lexeme != keyword) - { - return false; - } - else - { - function(theStream); - return true; - } -} - -bool RegisteredKeywordStreamOnly::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) -{ - if (strippedLexeme != keyword) - { - return false; - } - else - { - function(theStream); - return true; - } -} +RegisteredRegexBase::RegisteredRegexBase(const std::string& keyword): regex(std::regex(keyword)) {} -RegisteredRegex::RegisteredRegex(const std::string& keyword, const parsingFunction& function): regex(std::regex(keyword)), function{function} {} +RegisteredRegex::RegisteredRegex(const std::string& keyword, const parsingFunction& function): RegisteredRegexBase(keyword), function(function) {} bool RegisteredRegex::match(const std::string& lexeme, std::istream& theStream) { @@ -84,9 +29,7 @@ bool RegisteredRegex::match(const std::string& lexeme, std::istream& theStream) } } -bool RegisteredRegex::matchStripped(const std::string& lexeme, - const std::string& strippedLexeme, - std::istream& theStream) +bool RegisteredRegex::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { if (!std::regex_match(strippedLexeme, regex)) return false; @@ -98,7 +41,7 @@ bool RegisteredRegex::matchStripped(const std::string& lexeme, } -RegisteredRegexStreamOnly::RegisteredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): regex(std::regex(keyword)), function{function} {} +RegisteredRegexStreamOnly::RegisteredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function): RegisteredRegexBase(keyword), function(function) {} bool RegisteredRegexStreamOnly::match(const std::string& lexeme, std::istream& theStream) { @@ -111,9 +54,7 @@ bool RegisteredRegexStreamOnly::match(const std::string& lexeme, std::istream& t } } -bool RegisteredRegexStreamOnly::matchStripped(const std::string& lexeme, - const std::string& strippedLexeme, - std::istream& theStream) +bool RegisteredRegexStreamOnly::matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) { if (!std::regex_match(strippedLexeme, regex)) return false; @@ -139,25 +80,25 @@ void commonItems::absorbBOM(std::istream& theStream) void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunction& function) { - registered.emplace_back(std::make_unique(keyword, function)); + registeredRegexes.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunctionStreamOnly& function) { - registered.emplace_back(std::make_unique(keyword, function)); + registeredRegexes.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function) { - registered.emplace_back(std::make_unique(keyword, function)); + registeredRegexes.emplace_back(std::make_unique(keyword, function)); } void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunctionStreamOnly& function) { - registered.emplace_back(std::make_unique(keyword, function)); + registeredRegexes.emplace_back(std::make_unique(keyword, function)); } @@ -233,7 +174,7 @@ void commonItems::parser::parseFile(std::string_view filename) void commonItems::parser::clearRegisteredKeywords() noexcept { - std::vector>().swap(registered); + std::vector>().swap(registeredRegexes); } @@ -254,7 +195,7 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt const auto isLexemeQuoted = (strippedLexeme.size() < toReturn.size()); bool matched = false; - for (const auto& registered: registered) + for (const auto& registered: registeredRegexes) { if (registered->match(toReturn, theStream)) { @@ -264,7 +205,7 @@ std::optional commonItems::parser::getNextToken(std::istream& theSt } if (!matched && isLexemeQuoted) { - for (const auto& registered: registered) + for (const auto& registered: registeredRegexes) { if (registered->matchStripped(toReturn, strippedLexeme, theStream)) { diff --git a/Parser.h b/Parser.h index 7fd823ab..45c0ab4e 100644 --- a/Parser.h +++ b/Parser.h @@ -20,40 +20,21 @@ typedef std::function parsingFunctionStreamOnly; void absorbBOM(std::istream& theStream); -class Registered +class RegisteredRegexBase { + protected: + std::regex regex; public: - virtual ~Registered() = default; + RegisteredRegexBase(const std::string& keyword); + virtual ~RegisteredRegexBase() = default; virtual bool match(const std::string& lexeme, std::istream& theStream) = 0; virtual bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream) = 0; }; -class RegisteredKeyword: public Registered -{ - private: - std::string keyword; - parsingFunction function; - public: - RegisteredKeyword(const std::string& keyword, const parsingFunction& function); - bool match(const std::string& lexeme, std::istream& theStream); - bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); -}; - -class RegisteredKeywordStreamOnly: public Registered -{ - private: - std::string keyword; - parsingFunctionStreamOnly function; - public: - RegisteredKeywordStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function); - bool match(const std::string& lexeme, std::istream& theStream); - bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); -}; -class RegisteredRegex: public Registered +class RegisteredRegex: public RegisteredRegexBase { private: - std::regex regex; parsingFunction function; public: RegisteredRegex(const std::string& keyword, const parsingFunction& function); @@ -61,10 +42,9 @@ class RegisteredRegex: public Registered bool matchStripped(const std::string& lexeme, const std::string& strippedLexeme, std::istream& theStream); }; -class RegisteredRegexStreamOnly: public Registered +class RegisteredRegexStreamOnly: public RegisteredRegexBase { private: - std::regex regex; parsingFunctionStreamOnly function; public: RegisteredRegexStreamOnly(const std::string& keyword, const parsingFunctionStreamOnly& function); @@ -98,7 +78,7 @@ class parser private: - std::vector> registered; + std::vector> registeredRegexes; }; } // namespace commonItems