From 7b292ac1a97624f680e9a57347b34fac708beec1 Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Sun, 14 Feb 2021 20:07:14 -0500 Subject: [PATCH 1/3] liquid: update for v5 release --- lib/rouge/lexers/liquid.rb | 322 ++++++++++++++++++------------------- spec/visual/samples/liquid | 60 ++++--- 2 files changed, 192 insertions(+), 190 deletions(-) diff --git a/lib/rouge/lexers/liquid.rb b/lib/rouge/lexers/liquid.rb index 75ca9b3225..01498f2418 100644 --- a/lib/rouge/lexers/liquid.rb +++ b/lib/rouge/lexers/liquid.rb @@ -13,223 +13,154 @@ class Liquid < RegexLexer rule %r/[^\{]+/, Text rule %r/(\{%-?)(\s*)/ do - groups Punctuation, Text::Whitespace - push :tag_or_block + groups Comment::Preproc, Text::Whitespace + push :logic end rule %r/(\{\{-?)(\s*)/ do - groups Punctuation, Text::Whitespace + groups Comment::Preproc, Text::Whitespace push :output end rule %r/\{/, Text end - state :tag_or_block do + state :end_logic do + rule(/(\s*)(-?%\})/) do + groups Text::Whitespace, Comment::Preproc + reset_stack + end + + rule(/\n\s*/) do + token Text::Whitespace + if in_state? :liquid + pop! until state? :liquid + end + end + end + + state :end_output do + rule(/(\s*)(-?\}\})/) do + groups Text::Whitespace, Comment::Preproc + reset_stack + end + end + + state :liquid do + mixin :end_logic + mixin :logic + end + + state :logic do + rule %r/(liquid)\b/, Name::Tag, :liquid + # builtin logic blocks - rule %r/(if|elsif|unless|case)\b/, Keyword::Reserved, :condition - rule %r/(when)\b/, Keyword::Reserved, :when + rule %r/(if|elsif|unless|case)\b/, Name::Tag, :condition + rule %r/(when)\b/, Name::Tag, :value + rule %r/(else|ifchanged|end\w+)\b/, Name::Tag, :end_logic + rule %r/(break|continue)\b/, Keyword::Reserved, :end_logic - rule %r/(else)(\s*)(-?%\})/ do - groups Keyword::Reserved, Text::Whitespace, Punctuation - pop! + # builtin iteration blocks + rule %r/(for|tablerow)(\s+)(\S+)(\s+)(in)(\s+)/ do + groups Name::Tag, Text::Whitespace, Name::Variable, Text::Whitespace, + Name::Tag, Text::Whitespace + push :iteration_args end # other builtin blocks - rule %r/(capture|(?:in|de)crement)(\s+)([^\s%]+)(\s*)(-?%\})/ do - groups Name::Tag, Text::Whitespace, Name::Variable, Text::Whitespace, Punctuation - pop! + rule %r/(capture|(?:in|de)crement)(\s+)([a-zA-Z_](?:\w|-(?!%))*)/ do + groups Name::Tag, Text::Whitespace, Name::Variable + push :end_logic end rule %r/(comment)(\s*)(-?%\})/ do - groups Name::Tag, Text::Whitespace, Punctuation + groups Name::Tag, Text::Whitespace, Comment::Preproc push :comment end rule %r/(raw)(\s*)(-?%\})/ do - groups Name::Tag, Text::Whitespace, Punctuation + groups Name::Tag, Text::Whitespace, Comment::Preproc push :raw end - # end of block - rule %r/(end(?:if|unless|case))(\s*)(-?%\})/ do - groups Keyword::Reserved, Text::Whitespace, Punctuation - pop! - end - - rule %r/(end(?:[^\s%]+))(\s*)(-?%\})/ do - groups Name::Tag, Text::Whitespace, Punctuation - pop! - end - # builtin tags rule %r/(assign|echo)\b/, Name::Tag, :assign - rule %r/(include|render)\b/, Name::Tag, :include - rule %r/(cycle)(\s+)(?:([^\s:]*)(\s*)(:))?(\s*)/ do |m| + rule %r/(include|render)(\s+)([\/\w-]+(?:\.[\w-]+)+\b)?/ do + groups Name::Tag, Text::Whitespace, Text + push :include + end + + rule %r/(cycle)(\s+)(?:([\w-]+|'[^']*'|"[^"]*")(\s*)(:))?(\s*)/ do |m| token_class = case m[3] when %r/'[^']*'/ then Str::Single when %r/"[^"]*"/ then Str::Double else Name::Attribute end - groups Name::Tag, Text::Whitespace, token_class, Text::Whitespace, Punctuation, Text::Whitespace - - push :variable_tag_markup + push :tag_args end - # iteration - rule %r/ - (for|tablerow)(\s+) - ([\w-]+)(\s+) - (in)(\s+) - ( - (?: [^\s%,\|'"] | (?:"[^"]*"|'[^']*') )+ - )(\s*) - /x do |m| - groups Name::Tag, Text::Whitespace, Name::Variable, Text::Whitespace, - Name::Tag, Text::Whitespace - - token_class = case m[7] - when %r/'[^']*'/ then Str::Single - when %r/"[^"]*"/ then Str::Double - else - Name::Variable - end - token token_class, m[7] - token Text::Whitespace, m[8] - push :tag_markup - end - - # other tags or blocks - rule %r/([^\s%]+)(\s*)/ do - groups Name::Tag, Text::Whitespace - push :tag_markup - end + # custom tags or blocks + rule %r/(\w+)\b/, Name::Tag, :block_args end state :output do - rule %r/(\|)(\s*)([a-zA-Z_][^\s}\|:]*)/ do - groups Punctuation, Text::Whitespace, Name::Function + rule %r/(\|)(\s*)/ do + groups Punctuation, Text::Whitespace push :filters + push :filter end - mixin :end_of_tag - mixin :generic - end - - state :filters do - rule %r/(\|)(\s*)([a-zA-Z_][^\s%}\|:]*)/ do - groups Punctuation, Text::Whitespace, Name::Function - end - - mixin :end_of_tag - mixin :end_of_block - mixin :variable_param_markup + mixin :end_output + mixin :static + mixin :variable end state :condition do rule %r/([=!]=|[<>]=?)/, Operator rule %r/(and|or|contains)\b/, Operator::Word - mixin :end_of_block - mixin :generic - end - - state :when do - mixin :end_of_block - mixin :generic - end - - state :end_of_tag do - rule(/-?\}\}/) { token Punctuation; reset_stack } - end - - state :end_of_block do - rule(/-?%\}/) { token Punctuation; reset_stack } - end - - # states for unknown markup - state :param_markup do - mixin :whitespace - mixin :keyword - mixin :string - mixin :number - - rule %r/([^\s=:]+)(\s*)(=|:)/ do - groups Name::Attribute, Text::Whitespace, Operator - end - - rule %r/[,:]/, Punctuation + mixin :value end - state :default_param_markup do - mixin :param_markup - - rule %r/\S+/, Text - end - - state :variable_param_markup do - mixin :param_markup + state :value do + mixin :end_logic + mixin :static mixin :variable - - rule %r/\S+/, Text - end - - state :tag_markup do - rule %r/(reversed)\b/, Name::Attribute - - mixin :end_of_block - mixin :default_param_markup - end - - state :variable_tag_markup do - mixin :end_of_block - mixin :variable_param_markup end - # states for different values types - state :keyword do - rule %r/(false|true|nil)\b/, Keyword::Constant - end + state :iteration_args do + rule %r/(reversed|continue)\b/, Name::Attribute + rule %r/\(/, Punctuation, :range - state :variable do - rule %r/(empty|blank|forloop\.[^\s%}\|:]+)\b/, Name::Builtin - rule %r/\.(?=\w)|\[|\]/, Punctuation - rule %r/(first|last|size)\b/, Name::Function - rule %r/[a-zA-Z_][\w-]*\??/, Name::Variable + mixin :tag_args end - state :string do - rule %r/'[^']*'/, Str::Single - rule %r/"[^"]*"/, Str::Double - end + state :block_args do + rule %r/(\{\{-?)/, Comment::Preproc, :output_embed - state :number do - rule %r/-/, Operator - rule %r/\d+\.\d+/, Num::Float - rule %r/\d+/, Num::Integer - end + rule %r/(\|)(\s*)/ do + groups Punctuation, Text::Whitespace + push :filters + push :filter + end - state :generic do - mixin :whitespace - mixin :keyword - mixin :string - mixin :number - mixin :variable + mixin :tag_args end - state :whitespace do - rule %r/[ \t]+/, Text::Whitespace + state :tag_args do + mixin :end_logic + mixin :args end state :comment do rule %r/[^\{]+/, Comment rule %r/(\{%-?)(\s*)(endcomment)(\s*)(-?%\})/ do - groups Punctuation, Text::Whitespace, Name::Tag, Text::Whitespace, Punctuation + groups Comment::Preproc, Text::Whitespace, Name::Tag, Text::Whitespace, Comment::Preproc reset_stack end @@ -240,7 +171,7 @@ class Liquid < RegexLexer rule %r/[^\{]+/, Text rule %r/(\{%-?)(\s*)(endraw)(\s*)(-?%\})/ do - groups Punctuation, Text::Whitespace, Name::Tag, Text::Whitespace, Punctuation + groups Comment::Preproc, Text::Whitespace, Name::Tag, Text::Whitespace, Comment::Preproc reset_stack end @@ -249,36 +180,97 @@ class Liquid < RegexLexer state :assign do rule %r/=/, Operator + rule %r/\(/, Punctuation, :range - rule %r/(\|)(\s*)([a-zA-Z_][^\s%\|:]*)/ do - groups Punctuation, Text::Whitespace, Name::Function + rule %r/(\|)(\s*)/ do + groups Punctuation, Text::Whitespace push :filters + push :filter end - mixin :end_of_block - mixin :generic + mixin :value end state :include do - rule %r/(\{\{-?)(\s*)/ do + rule %r/(\{\{-?)/, Comment::Preproc, :output_embed + rule %r/(with|for|as)\b/, Keyword::Reserved + + mixin :tag_args + end + + state :output_embed do + rule %r/(\|)(\s*)([a-zA-Z_](?:\w|-(?!}))*)/ do + groups Punctuation, Text::Whitespace, Name::Function + end + + rule %r/-?\}\}/, Comment::Preproc, :pop! + + mixin :args + end + + state :range do + rule %r/\.\./, Punctuation + rule %r/\)/, Punctuation, :pop! + + mixin :whitespace + mixin :number + mixin :variable + end + + state :filters do + rule %r/(\|)(\s*)/ do groups Punctuation, Text::Whitespace - push :output_embed + push :filter end - rule %r/(with|for)\b/, Name::Tag - rule %r/[\/\w-]+(\.[\w-]+)+\b/, Text + mixin :end_logic + mixin :end_output + mixin :args + end - mixin :variable_tag_markup + state :filter do + rule %r/[a-zA-Z_](?:\w|-(?![%}]))*/, Name::Function, :pop! end - state :output_embed do - rule %r/(\|)(\s*)([a-zA-Z_][^\s}\|:]*)/ do - groups Punctuation, Text::Whitespace, Name::Function + state :args do + mixin :static + + rule %r/([a-zA-Z_][\w-]*)(\s*)(=|:)/ do + groups Name::Attribute, Text::Whitespace, Operator end - rule %r/-?\}\}/, Punctuation, :pop! + mixin :variable + end + + state :static do + rule %r/(false|true|nil)\b/, Keyword::Constant + rule %r/'[^']*'/, Str::Single + rule %r/"[^"]*"/, Str::Double + rule %r/[,:]/, Punctuation + + mixin :whitespace + mixin :number + end - mixin :variable_param_markup + state :whitespace do + rule %r/\s+/, Text::Whitespace + end + + state :number do + rule %r/-/, Operator + rule %r/\d+\.\d+/, Num::Float + rule %r/\d+/, Num::Integer + end + + state :variable do + rule %r/(\.)(\s*)(first|last|size)\b(?![?!\/])/ do + groups Punctuation, Text::Whitespace, Name::Function + end + + rule %r/\.(?= *\w)|\[|\]/, Punctuation + rule %r/(empty|blank|(for|tablerow)loop\.(parentloop\.)*\w+)\b(?![?!\/])/, Name::Builtin + rule %r/[a-zA-Z_][\w-]*\b-?(?![?!\/])/, Name::Variable + rule %r/\S+/, Text end end end diff --git a/spec/visual/samples/liquid b/spec/visual/samples/liquid index bf33219f68..ab075aac69 100644 --- a/spec/visual/samples/liquid +++ b/spec/visual/samples/liquid @@ -14,28 +14,26 @@ Just regular text - what happens? {% endcomment %} {% custom_tag param1: true param2 : nil %} -{% custom_block my="abc" c = false %} +{% custom_block my="abc" c = var %} Just usual {{liquid}}. {% endcustom_block %} {% another_tag "my string param" %} {{ variable | upcase }} -{{ var.field | textilize | markdownify }} +{{ var . field | textilize | markdownify }} {{ var.field.property | textilize | markdownify }} {{ -3.14 | abs }} {{ 'string' | truncate: 100 param='df"g' }} -{{ variable-nil? }} +{{ variable.null? }} {% capture name %} {{- title | downcase | slice: -3, 2 -}} {% endcapture %} -{%- assign life = 'infinite' | upcase -%} - -{% cycle '1', 2, var %} +{% cycle "width: 1em", 2, var %} {% cycle 'group1': '1', var, 2 %} -{% cycle group2: '1', var, 2 %} +{% cycle group2 : '1', var, 2 %} {% if a == 'B' %} Testing {{ some }} stuff. @@ -44,30 +42,30 @@ Testing {{ some }} stuff. {% else %} {% endif %} -{% unless a %} +{% unless a.empty? %} Some {{ output }} right here. {% else %} {% endunless %} -{% case a %} -{% when 'B' %} +{% case a.first %} +{% when 'B' or -1 %} Some {{ output }}! -{% when 'C' %} +{% when 'C', 4 %} Some other {{ output }}! {% else %} {% endcase %} -{% include dir/file.html param = 'example.com' param2 = object %} -{% include 'snippet', param: 'example.com', param2: object %} -{% include product_page with products[0] %} -{% include {{product_page | split: "." | first}} for products %} +{% include dir/file.html param="example.com" param2 = object.property %} +{% include 'snippet', param: 'example.com', param2 : object.property %} +{% include product_pages[3] with products[0] as product %} +{% include {{-product_page | split: "." | first-}} for products %} {% assign page_has_image = false %} -{% assign img_tag = '<' | append: 'img' %} +{%-assign img_tag = '<' | append: "img" | downcase-%} {% if link.object.content contains img_tag %} {% assign src = link.object.content | split: 'src="' %} {% assign src = src[1] | split: '"' | first %} - {% if src.size %} + {% if src.size > 0 %} {% assign page_has_image = true %} {% assign image_src = src | replace: '_small', '' | replace: '_compact', '' | replace: '_medium', '' | replace: '_large', '' | replace: '_grande', '' %} @@ -84,8 +82,8 @@ Some other {{ output }}! {% endif %} -{% tablerow page in site.pages %} - {% if page.layout == 'home' %} +{% tablerow page in site.pages cols:num limit:3 %} + {% if page.layout == 'home' and tablerowloop.col_first %} {{ page.excerpt }} {% endif %} {% endtablerow %} @@ -99,24 +97,25 @@ Some other {{ output }}! {% endfor %} {% for item in array reversed limit:2 %} - Item {{ forloop.index }}: {{ item.name }} + Item {{ forloop.index0 }}: {{ item.name }} {% endfor %} -{% for item in array offset:2 %} +{% for item in array, offset:continue %} {% increment var %} {% endfor %} -{% for item in array reversed %} +{% for item in site.data[path] reversed %} {% decrement var %} {% endfor %} -{% for i in (3..5) %} +{% assign range = (3 .. 5) %} +{% for i in range %} {% render "snippet", number: i %} {% endfor %} {% assign num- = 4 %} {% for i in (1..num-) %} - {%-if forloop.last-%}{{ i }}{%-endif-%} + {%-if forloop.first-%}{{ i }}{%-endif-%} {% endfor %} {% for char in 'The Quick Brown Fox' %} @@ -124,5 +123,16 @@ Some other {{ output }}! {% endfor %} {% for char in "Hello World" reversed %} - {% echo char | upcase %} + {% echo char | url_encode %} {% endfor %} + +{% liquid %} +{% liquid + assign size = "one two" | split: " " + for value in size + echo value + unless forloop.last + echo ", " + endunless + endfor +%} From c51504cf997d70a53c0908136981f0263894c7d6 Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Wed, 17 Feb 2021 14:31:19 -0500 Subject: [PATCH 2/3] liquid: add mimetype --- lib/rouge/lexers/liquid.rb | 1 + spec/lexers/liquid_spec.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/lib/rouge/lexers/liquid.rb b/lib/rouge/lexers/liquid.rb index 01498f2418..a92136325a 100644 --- a/lib/rouge/lexers/liquid.rb +++ b/lib/rouge/lexers/liquid.rb @@ -8,6 +8,7 @@ class Liquid < RegexLexer desc 'Liquid is a templating engine for Ruby (liquidmarkup.org)' tag 'liquid' filenames '*.liquid' + mimetypes 'text/html+liquid' state :root do rule %r/[^\{]+/, Text diff --git a/spec/lexers/liquid_spec.rb b/spec/lexers/liquid_spec.rb index ef431642db..11de72030e 100644 --- a/spec/lexers/liquid_spec.rb +++ b/spec/lexers/liquid_spec.rb @@ -10,5 +10,9 @@ it 'guesses by filename' do assert_guess :filename => 'file.liquid' end + + it 'guesses by mimetype' do + assert_guess mimetype: 'text/html+liquid' + end end end From 19620c0029733febb9d13c89044598b11ce39161 Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Fri, 19 Feb 2021 16:49:52 -0500 Subject: [PATCH 3/3] liquid: support inline comments --- lib/rouge/lexers/liquid.rb | 7 +++++++ spec/visual/samples/liquid | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/rouge/lexers/liquid.rb b/lib/rouge/lexers/liquid.rb index a92136325a..c54d700bce 100644 --- a/lib/rouge/lexers/liquid.rb +++ b/lib/rouge/lexers/liquid.rb @@ -38,6 +38,8 @@ class Liquid < RegexLexer pop! until state? :liquid end end + + mixin :whitespace end state :end_output do @@ -106,6 +108,8 @@ class Liquid < RegexLexer # custom tags or blocks rule %r/(\w+)\b/, Name::Tag, :block_args + + mixin :end_logic end state :output do @@ -231,6 +235,8 @@ class Liquid < RegexLexer state :filter do rule %r/[a-zA-Z_](?:\w|-(?![%}]))*/, Name::Function, :pop! + + mixin :whitespace end state :args do @@ -255,6 +261,7 @@ class Liquid < RegexLexer state :whitespace do rule %r/\s+/, Text::Whitespace + rule %r/#.*?(?=$|-?[}%]})/, Comment end state :number do diff --git a/spec/visual/samples/liquid b/spec/visual/samples/liquid index ab075aac69..414f57445f 100644 --- a/spec/visual/samples/liquid +++ b/spec/visual/samples/liquid @@ -119,15 +119,18 @@ Some other {{ output }}! {% endfor %} {% for char in 'The Quick Brown Fox' %} - {{ char | upcase }} + {{ char | upcase #| append: "inline comment" }} {% endfor %} {% for char in "Hello World" reversed %} {% echo char | url_encode %} -{% endfor %} +{% endfor + # inline comment %} {% liquid %} {% liquid + # multiline comment + ################### assign size = "one two" | split: " " for value in size echo value