diff --git a/grammars/yaml.cson b/grammars/yaml.cson index 12c4e46..f65ce21 100644 --- a/grammars/yaml.cson +++ b/grammars/yaml.cson @@ -16,19 +16,30 @@ 'Boxfile' ] 'firstLineMatch': '^(#cloud-config|---)' +'injections': + 'L:source.yaml - string.unquoted.block.yaml': + 'patterns': [ + { + 'match': '^\\s*?(\\t+)' + 'captures': + '1': + 'name': 'invalid.illegal.whitespace.yaml' + } + ] + 'source.yaml - (string.unquoted.yaml | string.unquoted.block.yaml)': + 'patterns': [ + { + # Captures any leading whitespace that the first pattern missed + 'match': '\\t+(?!\\s*$)' + 'name': 'invalid.illegal.whitespace.yaml' + } + ] 'patterns': [ { - 'include': '#erb' - } - { - 'include': '#comment' - } - { - 'match': '\\t+' - 'name': 'invalid.illegal.whitespace.yaml' + 'include': '#common' } { - 'match': '^---' + 'match': '^---$' 'name': 'punctuation.definition.directives.end.yaml' } { @@ -36,163 +47,40 @@ 'name': 'punctuation.definition.document.end.yaml' } { - # hello: > - # hello: | - 'begin': '^(\\s*)(?!-\\s*)(\\S+)\\s*(:)(?:\\s+((!)[^!\\s]+))?\\s+(?=\\||>)' + 'begin': '^(?:(---)\\s+)?(?=\\s*+(?!-\\s+)[^{\\[@#%&*>,:][^:]*$)' # Possible non-scalar 'beginCaptures': - '2': - 'name': 'entity.name.tag.yaml' - '3': - 'name': 'punctuation.separator.key-value.yaml' - '4': - 'name': 'keyword.other.tag.local.yaml' - '5': - 'name': 'punctuation.definition.tag.local.yaml' - 'end': '^(((?!$)(?!\\1\\s+))|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))' - 'contentName': 'string.unquoted.block.yaml' - 'patterns': [ - { - 'include': '#constants' - } - { - 'include': '#erb' - } - ] - } - { - # - > - # - | - # - hello: > - # - hello: | - 'begin': '^(\\s*)(?:(-)|(?:(?:(-)\\s*)?(\\S+)\\s*(:)))(?:\\s+((!)[^!\\s]+))?\\s+(?=\\||>)' - 'beginCaptures': - '2': - 'name': 'punctuation.definition.entry.yaml' - '3': - 'name': 'punctuation.definition.entry.yaml' - '4': - 'name': 'entity.name.tag.yaml' - '5': - 'name': 'punctuation.separator.key-value.yaml' - '6': - 'name': 'keyword.other.tag.local.yaml' - '7': - 'name': 'punctuation.definition.tag.local.yaml' - 'end': '^(((?!$)(?!\\1\\s+))|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))' - 'contentName': 'string.unquoted.block.yaml' + '1': + 'name': 'punctuation.definition.directives.end.yaml' + 'end': '(?=^\\.\\.\\.)' 'patterns': [ { - 'include': '#constants' - } - { - 'include': '#erb' + 'match': '\\G(.+)$' + 'captures': + '1': + 'patterns': [ + { + 'include': '#strings' + } + ] } - ] - } - { - # << : *variableToMerge - 'match': '(<<)\\s*(:)\\s+(.+)$' - 'captures': - '1': - 'name': 'entity.name.tag.merge.yaml' - '2': - 'name': 'punctuation.separator.key-value.yaml' - '3': - 'patterns': [ - { - 'include': '#variables' - } - ] - } - { - # - hello: - # look at me go: - # omap time: !!omap - 'begin': '(?>^(\\s*)(-)?\\s*)([^!{@#%&*>,\'"][^#\'"]*?)(:)\\s+((!!)omap)?' - 'beginCaptures': - '2': - 'name': 'punctuation.definition.entry.yaml' - '3': - 'name': 'entity.name.tag.yaml' - '4': - 'name': 'punctuation.separator.key-value.yaml' - '5': - 'name': 'keyword.other.omap.yaml' - '6': - 'name': 'punctuation.definition.tag.omap.yaml' - 'end': '^((?!\\1\\s+)|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+|#)))' - 'patterns': [ { - 'include': '#scalar-content' + 'include': '#common' } - ] - } - { - # - 'quoted': - # "quoted": - # "with omap": !!omap - 'begin': '^(\\s*)(-)?\\s*(?:((\')([^\']*?)(\'))|((")([^"]*?)(")))(:)\\s+((!!)omap)?' - 'beginCaptures': - '2': - 'name': 'punctuation.definition.entry.yaml' - '3': - 'name': 'string.quoted.single.yaml' - '4': - 'name': 'punctuation.definition.string.begin.yaml' - '5': - 'name': 'entity.name.tag.yaml' - '6': - 'name': 'punctuation.definition.string.end.yaml' - '7': - 'name': 'string.quoted.double.yaml' - '8': - 'name': 'punctuation.definition.string.begin.yaml' - '9': - 'name': 'entity.name.tag.yaml' - '10': - 'name': 'punctuation.definition.string.end.yaml' - '11': - 'name': 'punctuation.separator.key-value.yaml' - '12': - 'name': 'keyword.other.omap.yaml' - '13': - 'name': 'punctuation.definition.tag.omap.yaml' - 'end': '^((?!\\1\\s+)|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+|#)))' - 'patterns': [ { - 'include': '#scalar-content' + 'match': '.+' # Anything past the first non-scalar is invalid + 'name': 'invalid.illegal.content-after-non-scalar.yaml' } ] } { - # - stuff - # - !!omap - # - - 'begin': '^(\\s*)(-)\\s+(?:((!!)omap)|((!)[^!\\s]+)|(?![!@#%&*>,]))' - 'beginCaptures': - '2': - 'name': 'punctuation.definition.entry.yaml' - '3': - 'name': 'keyword.other.omap.yaml' - '4': - 'name': 'punctuation.definition.tag.omap.yaml' - '5': - 'name': 'keyword.other.tag.local.yaml' - '6': - 'name': 'punctuation.definition.tag.local.yaml' - 'end': '^((?!\\1\\s+)|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+|#)))' + 'begin': '(?=^\\s*\\S+.*$)' # Optional spaces, followed by actual content + 'end': '(?=^\\.\\.\\.)' 'patterns': [ { - 'include': '#scalar-content' + 'include': '#scalars' } ] } - { - 'include': '#variables' - } - { - 'include': '#strings' - } ] 'repository': 'comment': @@ -202,6 +90,15 @@ 'name': 'comment.line.number-sign.yaml' '2': 'name': 'punctuation.definition.comment.yaml' + 'common': + 'patterns': [ + { + 'include': '#erb' + } + { + 'include': '#comment' + } + ] 'constants': 'match': '(?<=\\s)(true|false|null)(?=\\s*$)' 'name': 'constant.language.yaml' @@ -327,17 +224,180 @@ ] } { - 'match': '[^\\s"\'\\n](?!\\s*#(?!{))([^#\\n]|((? + # hello: | + 'begin': '^(\\s*)(?!-\\s*)(\\S+)\\s*(:)(?:\\s+((!)[^!\\s]+))?\\s+(?=\\||>)' + 'beginCaptures': + '2': + 'name': 'entity.name.tag.yaml' + '3': + 'name': 'punctuation.separator.key-value.yaml' + '4': + 'name': 'keyword.other.tag.local.yaml' + '5': + 'name': 'punctuation.definition.tag.local.yaml' + 'end': '^(((?!$)(?!\\1\\s+))|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))' + 'contentName': 'string.unquoted.block.yaml' + 'patterns': [ + { + 'include': '#constants' + } + { + 'include': '#erb' + } + ] + } + { + # - > + # - | + # - hello: > + # - hello: | + 'begin': '^(\\s*)(?:(-)|(?:(?:(-)\\s*)?(\\S+)\\s*(:)))(?:\\s+((!)[^!\\s]+))?\\s+(?=\\||>)' + 'beginCaptures': + '2': + 'name': 'punctuation.definition.entry.yaml' + '3': + 'name': 'punctuation.definition.entry.yaml' + '4': + 'name': 'entity.name.tag.yaml' + '5': + 'name': 'punctuation.separator.key-value.yaml' + '6': + 'name': 'keyword.other.tag.local.yaml' + '7': + 'name': 'punctuation.definition.tag.local.yaml' + 'end': '^(((?!$)(?!\\1\\s+))|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))' + 'contentName': 'string.unquoted.block.yaml' + 'patterns': [ + { + 'include': '#constants' + } + { + 'include': '#erb' + } + ] + } + { + # << : *variableToMerge + 'match': '(<<)\\s*(:)\\s+(.+)$' + 'captures': + '1': + 'name': 'entity.name.tag.merge.yaml' + '2': + 'name': 'punctuation.separator.key-value.yaml' + '3': + 'patterns': [ + { + 'include': '#variables' + } + ] + } + { + # - hello: + # look at me go: + # omap time: !!omap + 'begin': '(?>^(\\s*)(-)?\\s*)([^!{@#%&*>,\'"][^#\'"]*?)(:)\\s+((!!)omap)?' + 'beginCaptures': + '2': + 'name': 'punctuation.definition.entry.yaml' + '3': + 'name': 'entity.name.tag.yaml' + '4': + 'name': 'punctuation.separator.key-value.yaml' + '5': + 'name': 'keyword.other.omap.yaml' + '6': + 'name': 'punctuation.definition.tag.omap.yaml' + 'end': '^((?!\\1\\s+)|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))|(?=\\s+#)' + 'patterns': [ + { + 'include': '#scalar-content' + } + ] + } + { + # - 'quoted': + # "quoted": + # "with omap": !!omap + 'begin': '^(\\s*)(-)?\\s*(?:((\')([^\']*?)(\'))|((")([^"]*?)(")))(:)\\s+((!!)omap)?' + 'beginCaptures': + '2': + 'name': 'punctuation.definition.entry.yaml' + '3': + 'name': 'string.quoted.single.yaml' + '4': + 'name': 'punctuation.definition.string.begin.yaml' + '5': + 'name': 'entity.name.tag.yaml' + '6': + 'name': 'punctuation.definition.string.end.yaml' + '7': + 'name': 'string.quoted.double.yaml' + '8': + 'name': 'punctuation.definition.string.begin.yaml' + '9': + 'name': 'entity.name.tag.yaml' + '10': + 'name': 'punctuation.definition.string.end.yaml' + '11': + 'name': 'punctuation.separator.key-value.yaml' + '12': + 'name': 'keyword.other.omap.yaml' + '13': + 'name': 'punctuation.definition.tag.omap.yaml' + 'end': '^((?!\\1\\s+)|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))|(?=\\s+#)' + 'patterns': [ + { + 'include': '#scalar-content' + } + ] + } + { + # - stuff + # - !!omap + # - + 'begin': '^(\\s*)(-)\\s+(?:((!!)omap)|((!)(?:[^!\\s]+|(?=\\s+#|$)))|(?![!@#%&*>,]))' + 'beginCaptures': + '2': + 'name': 'punctuation.definition.entry.yaml' + '3': + 'name': 'keyword.other.omap.yaml' + '4': + 'name': 'punctuation.definition.tag.omap.yaml' + '5': + 'name': 'keyword.other.tag.local.yaml' + '6': + 'name': 'punctuation.definition.tag.local.yaml' + 'end': '^((?!\\1\\s+)|(?=\\1\\s*(-|[^!@#%&*>,].*:\\s+)))|(?=\\s+#)' + 'patterns': [ + { + 'include': '#scalar-content' + } + ] + } + { + 'include': '#common' + } + { + 'match': '\\S.+' # Non-scalars not allowed + 'name': 'invalid.illegal.not-a-scalar.yaml' + } + ] 'scalar-content': 'patterns': [ { 'include': '#comment' } { - 'match': '!(?=\\s)' + 'match': '!(?=\\s(?!$))' 'name': 'punctuation.definition.tag.non-specific.yaml' } { @@ -349,13 +409,30 @@ { 'include': '#numeric' } + { + 'include': '#variables' + } { 'include': '#strings' } ] 'variables': - 'captures': - '1': - 'name': 'punctuation.definition.variable.yaml' - 'match': '(&|\\*)\\w+$' - 'name': 'variable.other.yaml' + 'patterns': [ + { + 'match': '(&)\\w+' + 'name': 'variable.other.anchor.yaml' + 'captures': + '1': + 'name': 'punctuation.definition.anchor.yaml' + } + { + 'match': '((\\*)\\w+)\\s*(.*)$' + 'captures': + '1': + 'name': 'variable.other.alias.yaml' + '2': + 'name': 'punctuation.definition.alias.yaml' + '3': + 'name': 'invalid.illegal.content-after-alias.yaml' + } + ] diff --git a/spec/yaml-spec.coffee b/spec/yaml-spec.coffee index 84b2307..8fd5b6c 100644 --- a/spec/yaml-spec.coffee +++ b/spec/yaml-spec.coffee @@ -109,187 +109,255 @@ describe "YAML grammar", -> expect(tokens[1]).toEqual value: "I am not \\escaped", scopes: ["source.yaml", "string.quoted.single.yaml"] expect(tokens[2]).toEqual value: "'", scopes: ["source.yaml", "string.quoted.single.yaml", "punctuation.definition.string.end.yaml"] - describe "text blocks", -> - it "parses simple content", -> + describe "non-scalars", -> + it "tokenizes one non-scalar and everything else as invalid", -> + lines = grammar.tokenizeLines """ + look at me + oh no error: + """ + expect(lines[0][0]).toEqual value: "look at me", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[1][0]).toEqual value: "oh no error:", scopes: ["source.yaml", "invalid.illegal.content-after-non-scalar.yaml"] + + it "tokenizes the non-scalar when it is on the same line as the directive end marker", -> + lines = grammar.tokenizeLines """ + --- look at me + oh no error: + """ + expect(lines[0][0]).toEqual value: "---", scopes: ["source.yaml", "punctuation.definition.directives.end.yaml"] + expect(lines[0][2]).toEqual value: "look at me", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[1][0]).toEqual value: "oh no error:", scopes: ["source.yaml", "invalid.illegal.content-after-non-scalar.yaml"] + + it "tokenizes the non-scalar when it is on the same line as the directive end marker", -> + lines = grammar.tokenizeLines """ + ---look at me + oh no error: + """ + expect(lines[0][0]).toEqual value: "---look at me", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[1][0]).toEqual value: "oh no error:", scopes: ["source.yaml", "invalid.illegal.content-after-non-scalar.yaml"] + + it "tokenizes non-scalars as invalid after the first scalar is encountered", -> + lines = grammar.tokenizeLines """ + key: value + oh no error + """ + expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][3]).toEqual value: "value", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[1][0]).toEqual value: "oh no error", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] + + it "resets when a document end marker is reached", -> + lines = grammar.tokenizeLines """ + --- + look at me + oh no error: + ... + look at me + oh no error: + """ + expect(lines[0][0]).toEqual value: "---", scopes: ["source.yaml", "punctuation.definition.directives.end.yaml"] + expect(lines[1][0]).toEqual value: "look at me", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[2][0]).toEqual value: "oh no error:", scopes: ["source.yaml", "invalid.illegal.content-after-non-scalar.yaml"] + expect(lines[3][0]).toEqual value: "...", scopes: ["source.yaml", "punctuation.definition.document.end.yaml"] + expect(lines[4][0]).toEqual value: "look at me", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[5][0]).toEqual value: "oh no error:", scopes: ["source.yaml", "invalid.illegal.content-after-non-scalar.yaml"] + + lines = grammar.tokenizeLines """ + --- + key: value + oh no error + ... + key: value + oh no error + """ + expect(lines[0][0]).toEqual value: "---", scopes: ["source.yaml", "punctuation.definition.directives.end.yaml"] + expect(lines[1][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[1][3]).toEqual value: "value", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[2][0]).toEqual value: "oh no error", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] + expect(lines[3][0]).toEqual value: "...", scopes: ["source.yaml", "punctuation.definition.document.end.yaml"] + expect(lines[4][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[4][3]).toEqual value: "value", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[5][0]).toEqual value: "oh no error", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] + + describe "text blocks", -> + it "parses simple content", -> + lines = grammar.tokenizeLines """ + key: | + content here + second line + """ + expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + + it "parses content with empty lines", -> + lines = grammar.tokenizeLines """ + key: | + content here + + second line + """ + expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + + it "parses keys with decimals", -> + lines = grammar.tokenizeLines """ + 2.0: | + content here + second line + """ + expect(lines[0][0]).toEqual value: "2.0", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + + it "properly parses through pound signs in blocks", -> + lines = grammar.tokenizeLines """ + key: | + # this is not a legit comment + unquoted block + ### this is just a markdown header + another unquoted block + """ + expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][0]).toEqual value: " # this is not a legit comment", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: " unquoted block", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: " ### this is just a markdown header", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[4][0]).toEqual value: " another unquoted block", scopes: ["source.yaml", "string.unquoted.block.yaml"] + + it "parses keys following blocks in sequences", -> + lines = grammar.tokenizeLines """ + - textblock: > + multiline + text + key with spaces: following text + """ + expect(lines[0][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] + expect(lines[0][2]).toEqual value: "textblock", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][0]).toEqual value: " multiline", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: " text", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[3][1]).toEqual value: "key with spaces", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[3][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[3][3]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[3][4]).toEqual value: "following text", scopes: ["source.yaml", "string.unquoted.yaml"] + + it "parses content even when not using | or >", -> + lines = grammar.tokenizeLines """ + - textblock: + multiline + text + key: following text + """ + expect(lines[0][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] + expect(lines[0][2]).toEqual value: "textblock", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][1]).toEqual value: "multiline", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[2][1]).toEqual value: "text", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[3][0]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[3][1]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[3][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[3][3]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[3][4]).toEqual value: "following text", scopes: ["source.yaml", "string.unquoted.yaml"] + + describe "parses content with unindented empty lines", -> + it "ending the content", -> lines = grammar.tokenizeLines """ key: | content here + second line """ expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[0][3]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - it "parses content with empty lines", -> + it "ending with new element", -> lines = grammar.tokenizeLines """ key: | content here second line + other: hi """ expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[0][3]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[4][0]).toEqual value: "other", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[4][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[4][2]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[4][3]).toEqual value: "hi", scopes: ["source.yaml", "string.unquoted.yaml"] - it "parses keys with decimals", -> - lines = grammar.tokenizeLines """ - 2.0: | - content here - second line - """ - expect(lines[0][0]).toEqual value: "2.0", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - - it "properly parses through pound signs in blocks", -> + it "ending with new element, part of list", -> lines = grammar.tokenizeLines """ - key: | - # this is not a legit comment - unquoted block - ### this is just a markdown header - another unquoted block - """ - expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[1][0]).toEqual value: " # this is not a legit comment", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: " unquoted block", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: " ### this is just a markdown header", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[4][0]).toEqual value: " another unquoted block", scopes: ["source.yaml", "string.unquoted.block.yaml"] + - key: | + content here - it "parses keys following blocks in sequences", -> - lines = grammar.tokenizeLines """ - - textblock: > - multiline - text - key with spaces: following text + second line + - other: hi """ expect(lines[0][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] - expect(lines[0][2]).toEqual value: "textblock", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[0][2]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] expect(lines[0][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[1][0]).toEqual value: " multiline", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: " text", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[3][1]).toEqual value: "key with spaces", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[3][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[3][3]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[3][4]).toEqual value: "following text", scopes: ["source.yaml", "string.unquoted.yaml"] - - it "parses content even when not using | or >", -> + expect(lines[0][5]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[4][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] + expect(lines[4][1]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[4][2]).toEqual value: "other", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[4][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[4][4]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[4][5]).toEqual value: "hi", scopes: ["source.yaml", "string.unquoted.yaml"] + + it "ending with twice unindented new element", -> lines = grammar.tokenizeLines """ - - textblock: - multiline - text - key: following text - """ - expect(lines[0][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] - expect(lines[0][2]).toEqual value: "textblock", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[0][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[1][1]).toEqual value: "multiline", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[2][1]).toEqual value: "text", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[3][0]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[3][1]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[3][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[3][3]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[3][4]).toEqual value: "following text", scopes: ["source.yaml", "string.unquoted.yaml"] - - describe "parses content with unindented empty lines", -> - it "ending the content", -> - lines = grammar.tokenizeLines """ + root: key: | content here second line - """ - expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[0][3]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - - it "ending with new element", -> - lines = grammar.tokenizeLines """ + other: hi + """ + expect(lines[1][1]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[1][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][4]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[4][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[5][0]).toEqual value: "other", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[5][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[5][2]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[5][3]).toEqual value: "hi", scopes: ["source.yaml", "string.unquoted.yaml"] + + it "ending with an indented comment", -> + lines = grammar.tokenizeLines """ + root: key: | content here second line - other: hi - """ - expect(lines[0][0]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[0][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[0][3]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[4][0]).toEqual value: "other", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[4][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[4][2]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[4][3]).toEqual value: "hi", scopes: ["source.yaml", "string.unquoted.yaml"] - - it "ending with new element, part of list", -> - lines = grammar.tokenizeLines """ - - key: | - content here - - second line - - other: hi - """ - expect(lines[0][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] - expect(lines[0][2]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[0][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[0][5]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[1][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[4][0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] - expect(lines[4][1]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[4][2]).toEqual value: "other", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[4][3]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[4][4]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[4][5]).toEqual value: "hi", scopes: ["source.yaml", "string.unquoted.yaml"] - - it "ending with twice unindented new element", -> - lines = grammar.tokenizeLines """ - root: - key: | - content here - - second line - other: hi - """ - expect(lines[1][1]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[1][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[1][4]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[4][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[5][0]).toEqual value: "other", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[5][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[5][2]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[5][3]).toEqual value: "hi", scopes: ["source.yaml", "string.unquoted.yaml"] - - it "ending with an indented comment", -> - lines = grammar.tokenizeLines """ - root: - key: | - content here - - second line - # hi - """ - expect(lines[1][1]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] - expect(lines[1][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] - expect(lines[1][4]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[2][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[3][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[4][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] - expect(lines[5][0]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[5][1]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] - expect(lines[5][2]).toEqual value: " hi", scopes: ["source.yaml", "comment.line.number-sign.yaml"] + # hi + """ + expect(lines[1][1]).toEqual value: "key", scopes: ["source.yaml", "entity.name.tag.yaml"] + expect(lines[1][2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] + expect(lines[1][4]).toEqual value: "|", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[2][0]).toEqual value: " content here", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[3][0]).toEqual value: "", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[4][0]).toEqual value: " second line", scopes: ["source.yaml", "string.unquoted.block.yaml"] + expect(lines[5][0]).toEqual value: " ", scopes: ["source.yaml"] + expect(lines[5][1]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] + expect(lines[5][2]).toEqual value: " hi", scopes: ["source.yaml", "comment.line.number-sign.yaml"] it "does not confuse keys and strings", -> {tokens} = grammar.tokenizeLine("- 'Section 2.4: 3, 6abc, 12ab, 30, 32a'") @@ -359,7 +427,7 @@ describe "YAML grammar", -> expect(lines[2][2]).toEqual value: " ", scopes: ["source.yaml"] expect(lines[2][3]).toEqual value: "th{ree}", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[3][0]).toEqual value: "fourth:invalid", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[3][0]).toEqual value: "fourth:invalid", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] it "parses quoted keys", -> lines = grammar.tokenizeLines """ @@ -416,9 +484,9 @@ describe "YAML grammar", -> expect(lines[1][0]).toEqual value: "second", scopes: ["source.yaml", "entity.name.tag.yaml"] expect(lines[1][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] expect(lines[1][2]).toEqual value: " ", scopes: ["source.yaml"] - expect(lines[1][3]).toEqual value: "2nd ", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[1][4]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] - expect(lines[1][5]).toEqual value: "bar", scopes: ["source.yaml", "comment.line.number-sign.yaml"] + expect(lines[1][3]).toEqual value: "2nd", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[1][5]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] + expect(lines[1][6]).toEqual value: "bar", scopes: ["source.yaml", "comment.line.number-sign.yaml"] expect(lines[2][0]).toEqual value: "third", scopes: ["source.yaml", "entity.name.tag.yaml"] expect(lines[2][1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] @@ -434,26 +502,19 @@ describe "YAML grammar", -> lines = grammar.tokenizeLines """ multiline: # comment! - This should still be a string # another comment! - Ditto - # Guess what this is - String + This should still be a string # comment # comment """ expect(lines[0][3]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] - expect(lines[1][1]).toEqual value: "This should still be a string ", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[1][2]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] - expect(lines[2][1]).toEqual value: "Ditto", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[3][1]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] - expect(lines[4][1]).toEqual value: "String", scopes: ["source.yaml", "string.unquoted.yaml"] - expect(lines[5][0]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] + expect(lines[1][1]).toEqual value: "This should still be a string", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(lines[2][0]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] it "does not confuse keys and comments", -> {tokens} = grammar.tokenizeLine("- Entry 2 # This colon breaks syntax highlighting: see?") expect(tokens[0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] - expect(tokens[3]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] - expect(tokens[4]).toEqual value: " This colon breaks syntax highlighting: see?", scopes: ["source.yaml", "comment.line.number-sign.yaml"] + expect(tokens[4]).toEqual value: "#", scopes: ["source.yaml", "comment.line.number-sign.yaml", "punctuation.definition.comment.yaml"] + expect(tokens[5]).toEqual value: " This colon breaks syntax highlighting: see?", scopes: ["source.yaml", "comment.line.number-sign.yaml"] it "does not confuse keys and unquoted strings", -> {tokens} = grammar.tokenizeLine("- { role: common }") @@ -527,19 +588,19 @@ describe "YAML grammar", -> expect(tokens[0]).toEqual value: "<<", scopes: ["source.yaml", "entity.name.tag.merge.yaml"] expect(tokens[1]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] expect(tokens[2]).toEqual value: " ", scopes: ["source.yaml"] - expect(tokens[3]).toEqual value: "*", scopes: ["source.yaml", "variable.other.yaml", "punctuation.definition.variable.yaml"] - expect(tokens[4]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.yaml"] + expect(tokens[3]).toEqual value: "*", scopes: ["source.yaml", "variable.other.alias.yaml", "punctuation.definition.alias.yaml"] + expect(tokens[4]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.alias.yaml"] {tokens} = grammar.tokenizeLine "<< : *variable" expect(tokens[0]).toEqual value: "<<", scopes: ["source.yaml", "entity.name.tag.merge.yaml"] expect(tokens[1]).toEqual value: " ", scopes: ["source.yaml"] expect(tokens[2]).toEqual value: ":", scopes: ["source.yaml", "punctuation.separator.key-value.yaml"] expect(tokens[3]).toEqual value: " ", scopes: ["source.yaml"] - expect(tokens[4]).toEqual value: "*", scopes: ["source.yaml", "variable.other.yaml", "punctuation.definition.variable.yaml"] - expect(tokens[5]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.yaml"] + expect(tokens[4]).toEqual value: "*", scopes: ["source.yaml", "variable.other.alias.yaml", "punctuation.definition.alias.yaml"] + expect(tokens[5]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.alias.yaml"] {tokens} = grammar.tokenizeLine "<<:*variable" - expect(tokens[0]).toEqual value: "<<:*variable", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(tokens[0]).toEqual value: "<<:*variable", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] it "parses local tags", -> {tokens} = grammar.tokenizeLine "multiline: !something >" @@ -552,10 +613,11 @@ describe "YAML grammar", -> expect(tokens[3]).toEqual value: "tag", scopes: ["source.yaml", "keyword.other.tag.local.yaml"] {tokens} = grammar.tokenizeLine "- !" - expect(tokens[0]).toEqual value: "- !", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(tokens[0]).toEqual value: "-", scopes: ["source.yaml", "punctuation.definition.entry.yaml"] + expect(tokens[2]).toEqual value: "!", scopes: ["source.yaml", "keyword.other.tag.local.yaml", "punctuation.definition.tag.local.yaml"] {tokens} = grammar.tokenizeLine "- !!" - expect(tokens[0]).toEqual value: "- !!", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(tokens[0]).toEqual value: "- !!", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] it "parses the !!omap directive", -> {tokens} = grammar.tokenizeLine "hello: !!omap" @@ -577,7 +639,7 @@ describe "YAML grammar", -> expect(tokens[8]).toEqual value: "omap", scopes: ["source.yaml", "keyword.other.omap.yaml"] {tokens} = grammar.tokenizeLine "hello:!!omap" - expect(tokens[0]).toEqual value: "hello:!!omap", scopes: ["source.yaml", "string.unquoted.yaml"] + expect(tokens[0]).toEqual value: "hello:!!omap", scopes: ["source.yaml", "invalid.illegal.not-a-scalar.yaml"] it "parses dates in YYYY-MM-DD format", -> {tokens} = grammar.tokenizeLine "- date: 2001-01-01" @@ -678,23 +740,35 @@ describe "YAML grammar", -> describe "variables", -> it "tokenizes them", -> - {tokens} = grammar.tokenizeLine "&variable" - expect(tokens[0]).toEqual value: "&", scopes: ["source.yaml", "variable.other.yaml", "punctuation.definition.variable.yaml"] - expect(tokens[1]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.yaml"] + {tokens} = grammar.tokenizeLine "key: &variable" + expect(tokens[3]).toEqual value: "&", scopes: ["source.yaml", "variable.other.anchor.yaml", "punctuation.definition.anchor.yaml"] + expect(tokens[4]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.anchor.yaml"] + + {tokens} = grammar.tokenizeLine "key: *variable" + expect(tokens[3]).toEqual value: "*", scopes: ["source.yaml", "variable.other.alias.yaml", "punctuation.definition.alias.yaml"] + expect(tokens[4]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.alias.yaml"] - {tokens} = grammar.tokenizeLine "*variable" - expect(tokens[0]).toEqual value: "*", scopes: ["source.yaml", "variable.other.yaml", "punctuation.definition.variable.yaml"] - expect(tokens[1]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.yaml"] + {tokens} = grammar.tokenizeLine "key: &v3ryc001" + expect(tokens[3]).toEqual value: "&", scopes: ["source.yaml", "variable.other.anchor.yaml", "punctuation.definition.anchor.yaml"] + expect(tokens[4]).toEqual value: "v3ryc001", scopes: ["source.yaml", "variable.other.anchor.yaml"] - {tokens} = grammar.tokenizeLine "&v3ryc001" - expect(tokens[0]).toEqual value: "&", scopes: ["source.yaml", "variable.other.yaml", "punctuation.definition.variable.yaml"] - expect(tokens[1]).toEqual value: "v3ryc001", scopes: ["source.yaml", "variable.other.yaml"] + {tokens} = grammar.tokenizeLine "key: & variable" + expect(tokens[3]).toEqual value: "& variable", scopes: ["source.yaml", "string.unquoted.yaml"] - {tokens} = grammar.tokenizeLine "& variable" - expect(tokens[0]).toEqual value: "& variable", scopes: ["source.yaml", "string.unquoted.yaml"] + {tokens} = grammar.tokenizeLine "key: * variable" + expect(tokens[3]).toEqual value: "* variable", scopes: ["source.yaml", "string.unquoted.yaml"] - {tokens} = grammar.tokenizeLine "&variable hey" - expect(tokens[0]).toEqual value: "&variable hey", scopes: ["source.yaml", "string.unquoted.yaml"] + {tokens} = grammar.tokenizeLine "key: &variable hey" + expect(tokens[3]).toEqual value: "&", scopes: ["source.yaml", "variable.other.anchor.yaml", "punctuation.definition.anchor.yaml"] + expect(tokens[4]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.anchor.yaml"] + expect(tokens[5]).toEqual value: " ", scopes: ["source.yaml"] + expect(tokens[6]).toEqual value: "hey", scopes: ["source.yaml", "string.unquoted.yaml"] + + {tokens} = grammar.tokenizeLine "key: *variable hey" + expect(tokens[3]).toEqual value: "*", scopes: ["source.yaml", "variable.other.alias.yaml", "punctuation.definition.alias.yaml"] + expect(tokens[4]).toEqual value: "variable", scopes: ["source.yaml", "variable.other.alias.yaml"] + expect(tokens[5]).toEqual value: " ", scopes: ["source.yaml"] + expect(tokens[6]).toEqual value: "hey", scopes: ["source.yaml", "invalid.illegal.content-after-alias.yaml"] describe "constants", -> it "tokenizes true, false, and null as constants", -> @@ -748,6 +822,39 @@ describe "YAML grammar", -> expect(lines[9][0]).toEqual value: "...", scopes: ["source.yaml", "punctuation.definition.document.end.yaml"] describe "tabs", -> - it "marks them as invalid", -> + it "tokenizes tabs used as indentation as invalid", -> {tokens} = grammar.tokenizeLine "\t\ttabs:" - expect(tokens[0]).toEqual value: '\t\t', scopes: ['source.yaml', 'invalid.illegal.whitespace.yaml'] + expect(tokens[0]).toEqual value: "\t\t", scopes: ["source.yaml", "invalid.illegal.whitespace.yaml"] + + {tokens} = grammar.tokenizeLine " \ttabs:" + expect(tokens[0]).toEqual value: " ", scopes: ["source.yaml"] + expect(tokens[1]).toEqual value: "\t", scopes: ["source.yaml", "invalid.illegal.whitespace.yaml"] + + {tokens} = grammar.tokenizeLine " \t \ttabs:" + expect(tokens[0]).toEqual value: " ", scopes: ["source.yaml"] + expect(tokens[1]).toEqual value: "\t", scopes: ["source.yaml", "invalid.illegal.whitespace.yaml"] + expect(tokens[2]).toEqual value: " ", scopes: ["source.yaml"] + expect(tokens[3]).toEqual value: "\t", scopes: ["source.yaml", "invalid.illegal.whitespace.yaml"] + + lines = grammar.tokenizeLines """ + multiline: a + \ttab + """ + expect(lines[1][1]).toEqual value: "\t", scopes: ["source.yaml", "invalid.illegal.whitespace.yaml"] + + it "does not tokenize tabs in values as invalid", -> + {tokens} = grammar.tokenizeLine "key: \tvalue1\tvalue2" + expect(tokens[2]).toEqual value: " \t", scopes: ["source.yaml"] + expect(tokens[3]).toEqual value: "value1\tvalue2", scopes: ["source.yaml", "string.unquoted.yaml"] + + it "does not tokenize trailing tabs as invalid", -> + {tokens} = grammar.tokenizeLine "hi\t" + expect(tokens[1]).toEqual value: "\t", scopes: ["source.yaml"] + + it "does not tokenize tabs in explicit multiline scalars as invalid", -> + lines = grammar.tokenizeLines """ + multiline: > + one + \ttwo + """ + expect(lines[2][0]).toEqual value: " \ttwo", scopes: ["source.yaml", "string.unquoted.block.yaml"]