Implement constants, keywords, raw, builtin tags, etc

src/liquid/liquid.test.ts

@@ -32,11 +32,11 @@ testTokenization(
 					{ startIndex: 0, type: 'delimiter.html' },
 					{ startIndex: 1, type: 'tag.html' },
 					{ startIndex: 3, type: 'delimiter.html' },
-					{ startIndex: 4, type: 'delimiter.liquid' },
+					{ startIndex: 4, type: 'delimiter.output.liquid' },
 					{ startIndex: 6, type: '' },
 					{ startIndex: 7, type: 'variable.liquid' },
 					{ startIndex: 12, type: '' },
-					{ startIndex: 13, type: 'delimiter.liquid' },
+					{ startIndex: 13, type: 'delimiter.output.liquid' },
 					{ startIndex: 15, type: 'delimiter.html' },
 					{ startIndex: 17, type: 'tag.html' },
 					{ startIndex: 19, type: 'delimiter.html' }
@@ -52,17 +52,18 @@ testTokenization(
 					{ startIndex: 0, type: 'delimiter.html' },
 					{ startIndex: 1, type: 'tag.html' },
 					{ startIndex: 3, type: 'delimiter.html' },
-					{ startIndex: 4, type: 'delimiter.liquid' },
+					{ startIndex: 4, type: 'delimiter.output.liquid' },
 					{ startIndex: 6, type: '' },
 					{ startIndex: 7, type: 'number.liquid' },
 					{ startIndex: 17, type: '' },
-					{ startIndex: 20, type: 'variable.liquid' },
+					{ startIndex: 20, type: 'predefined.liquid' },
 					{ startIndex: 25, type: '' },
-					{ startIndex: 28, type: 'variable.liquid' },
+					{ startIndex: 28, type: 'predefined.liquid' },
+					{ startIndex: 35, type: 'variable.liquid' },
 					{ startIndex: 36, type: '' },
 					{ startIndex: 37, type: 'string.liquid' },
 					{ startIndex: 41, type: '' },
-					{ startIndex: 43, type: 'delimiter.liquid' },
+					{ startIndex: 43, type: 'delimiter.output.liquid' },
 					{ startIndex: 45, type: 'delimiter.html' },
 					{ startIndex: 47, type: 'tag.html' },
 					{ startIndex: 49, type: 'delimiter.html' }
@@ -70,7 +71,7 @@ testTokenization(
 			}
 		],
 
-		// Tag
+		// Simple Tag
 		[
 			{
 				line: '<div>{% render "files/file123.html" %}</div>',
@@ -78,13 +79,13 @@ testTokenization(
 					{ startIndex: 0, type: 'delimiter.html' },
 					{ startIndex: 1, type: 'tag.html' },
 					{ startIndex: 4, type: 'delimiter.html' },
-					{ startIndex: 5, type: 'delimiter.output.liquid' },
+					{ startIndex: 5, type: 'delimiter.tag.liquid' },
 					{ startIndex: 7, type: '' },
-					{ startIndex: 8, type: 'variable.liquid' },
+					{ startIndex: 8, type: 'predefined.liquid' },
 					{ startIndex: 14, type: '' },
 					{ startIndex: 15, type: 'string.liquid' },
 					{ startIndex: 35, type: '' },
-					{ startIndex: 36, type: 'delimiter.liquid' },
+					{ startIndex: 36, type: 'delimiter.tag.liquid' },
 					{ startIndex: 38, type: 'delimiter.html' },
 					{ startIndex: 40, type: 'tag.html' },
 					{ startIndex: 43, type: 'delimiter.html' }
@@ -92,7 +93,78 @@ testTokenization(
 			}
 		],
 
-		// Handlebars comment
+		// Tag with drop
+		[
+			{
+				line: '<div>{{ thing.other_thing }}</div>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: 'delimiter.html' },
+					{ startIndex: 5, type: 'delimiter.output.liquid' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'variable.liquid' },
+					{ startIndex: 13, type: '' },
+					{ startIndex: 14, type: 'variable.liquid' },
+					{ startIndex: 25, type: '' },
+					{ startIndex: 26, type: 'delimiter.output.liquid' },
+					{ startIndex: 28, type: 'delimiter.html' },
+					{ startIndex: 30, type: 'tag.html' },
+					{ startIndex: 33, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// If tag / keywords / block style tags
+		[
+			{
+				line:
+					'<div>{% if true=false %}<div>True</div>{% else %}<div>False</div>{% endif %}</div>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: 'delimiter.html' },
+					{ startIndex: 5, type: 'delimiter.tag.liquid' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'predefined.liquid' },
+					{ startIndex: 10, type: '' },
+					{ startIndex: 11, type: 'keyword.liquid' },
+					{ startIndex: 15, type: '' },
+					{ startIndex: 16, type: 'keyword.liquid' },
+					{ startIndex: 21, type: '' },
+					{ startIndex: 22, type: 'delimiter.tag.liquid' },
+					{ startIndex: 24, type: 'delimiter.html' },
+					{ startIndex: 25, type: 'tag.html' },
+					{ startIndex: 28, type: 'delimiter.html' },
+					{ startIndex: 29, type: '' },
+					{ startIndex: 33, type: 'delimiter.html' },
+					{ startIndex: 35, type: 'tag.html' },
+					{ startIndex: 38, type: 'delimiter.html' },
+					{ startIndex: 39, type: 'delimiter.tag.liquid' },
+					{ startIndex: 41, type: '' },
+					{ startIndex: 42, type: 'predefined.liquid' },
+					{ startIndex: 46, type: '' },
+					{ startIndex: 47, type: 'delimiter.tag.liquid' },
+					{ startIndex: 49, type: 'delimiter.html' },
+					{ startIndex: 50, type: 'tag.html' },
+					{ startIndex: 53, type: 'delimiter.html' },
+					{ startIndex: 54, type: '' },
+					{ startIndex: 59, type: 'delimiter.html' },
+					{ startIndex: 61, type: 'tag.html' },
+					{ startIndex: 64, type: 'delimiter.html' },
+					{ startIndex: 65, type: 'delimiter.tag.liquid' },
+					{ startIndex: 67, type: '' },
+					{ startIndex: 68, type: 'predefined.liquid' },
+					{ startIndex: 73, type: '' },
+					{ startIndex: 74, type: 'delimiter.tag.liquid' },
+					{ startIndex: 76, type: 'delimiter.html' },
+					{ startIndex: 78, type: 'tag.html' },
+					{ startIndex: 81, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Comment tag
 		[
 			{
 				line: '<div>Anything you put between {% comment %} and {% endcomment %} tags</div>',
@@ -110,6 +182,30 @@ testTokenization(
 					{ startIndex: 74, type: 'delimiter.html' }
 				]
 			}
+		],
+
+		// Raw tag
+		[
+			{
+				line:
+					'<div>Everything here should be escaped {% raw %} In Handlebars, {{ this }} will be HTML-escaped, but {{{ that }}} will not. {% endraw %}</div>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: 'delimiter.html' },
+					{ startIndex: 5, type: '' },
+					{ startIndex: 39, type: 'delimiter.tag.liquid' },
+					{ startIndex: 41, type: '' },
+					{ startIndex: 42, type: 'delimiter.tag.liquid' },
+					{ startIndex: 48, type: '' },
+					{ startIndex: 124, type: 'delimiter.tag.liquid' },
+					{ startIndex: 126, type: '' },
+					{ startIndex: 134, type: 'delimiter.tag.liquid' },
+					{ startIndex: 136, type: 'delimiter.html' },
+					{ startIndex: 138, type: 'tag.html' },
+					{ startIndex: 141, type: 'delimiter.html' }
+				]
+			}
 		]
 	]
 );

src/liquid/liquid.ts

@@ -27,9 +27,7 @@ const EMPTY_ELEMENTS: string[] = [
 export const conf: languages.LanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
 
-	// comments: {
-	// 	blockComment: ['{{!--', '--}}']
-	// },
+	// TODO support if,else,elseif,for,in and other built in keywords
 
 	brackets: [
 		['<!--', '-->'],
@@ -79,14 +77,100 @@ export const conf: languages.LanguageConfiguration = {
 export const language = <languages.IMonarchLanguage>{
 	defaultToken: '',
 	tokenPostfix: '',
-	// ignoreCase: true,
+
+	builtinTags: [
+		'if',
+		'else',
+		'elseif',
+		'endif',
+		'render',
+		'assign',
+		'capture',
+		'endcapture',
+		'case',
+		'endcase',
+		'comment',
+		'endcomment',
+		'cycle',
+		'decrement',
+		'for',
+		'endfor',
+		'include',
+		'increment',
+		'layout',
+		'raw',
+		'endraw',
+		'render',
+		'tablerow',
+		'endtablerow',
+		'unless',
+		'endunless'
+	],
+
+	builtinFilters: [
+		'abs',
+		'append',
+		'at_least',
+		'at_most',
+		'capitalize',
+		'ceil',
+		'compact',
+		'date',
+		'default',
+		'divided_by',
+		'downcase',
+		'escape',
+		'escape_once',
+		'first',
+		'floor',
+		'join',
+		'json',
+		'last',
+		'lstrip',
+		'map',
+		'minus',
+		'modulo',
+		'newline_to_br',
+		'plus',
+		'prepend',
+		'remove',
+		'remove_first',
+		'replace',
+		'replace_first',
+		'reverse',
+		'round',
+		'rstrip',
+		'size',
+		'slice',
+		'sort',
+		'sort_natural',
+		'split',
+		'strip',
+		'strip_html',
+		'strip_newlines',
+		'times',
+		'truncate',
+		'truncatewords',
+		'uniq',
+		'upcase',
+		'url_decode',
+		'url_encode',
+		'where'
+	],
+
+	constants: ['true', 'false'],
+	operators: ['==', '!=', '>', '<', '>=', '<='],
+
+	symbol: /[=><!]+/,
+	identifier: /[a-zA-Z_][\w]*/,
 
 	// The main tokenizer for our languages
 	tokenizer: {
 		root: [
 			[/\{\%\s*comment\s*\%\}/, 'comment.start.liquid', '@comment'],
-			[/\{\{/, { token: '@rematch', switchTo: '@liquidInSimpleState.root' }],
-			[/\{\%/, { token: '@rematch', switchTo: '@liquidInSimpleState.root' }],
+			// [/\{\%\s*raw\s*\%\}/, 'delimiter.tag', '@raw'],
+			[/\{\{/, { token: '@rematch', switchTo: '@liquidState.root' }],
+			[/\{\%/, { token: '@rematch', switchTo: '@liquidState.root' }],
 			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
 			[/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
 			[/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
@@ -105,14 +189,14 @@ export const language = <languages.IMonarchLanguage>{
 				/\{\{/,
 				{
 					token: '@rematch',
-					switchTo: '@liquidInSimpleState.otherTag'
+					switchTo: '@liquidState.otherTag'
 				}
 			],
 			[
-				/\{%/,
+				/\{\%/,
 				{
 					token: '@rematch',
-					switchTo: '@liquidInSimpleState.otherTag'
+					switchTo: '@liquidState.otherTag'
 				}
 			],
 			[/\/?>/, 'delimiter.html', '@pop'],
@@ -123,25 +207,48 @@ export const language = <languages.IMonarchLanguage>{
 			[/[ \t\r\n]+/] // whitespace
 		],
 
-		liquidInSimpleState: [
-			[/\{\{/, 'delimiter.liquid'],
-			[/\}\}/, { token: 'delimiter.liquid', switchTo: '@$S2.$S3' }],
-			[/\{\%/, 'delimiter.output.liquid'],
-			[/\%\}/, { token: 'delimiter.liquid', switchTo: '@$S2.$S3' }],
+		liquidState: [
+			[/\{\{/, 'delimiter.output.liquid'],
+			[/\}\}/, { token: 'delimiter.output.liquid', switchTo: '@$S2.$S3' }],
+			[/\{\%/, 'delimiter.tag.liquid'],
+			[/raw\s*\%\}/, 'delimiter.tag.liquid', '@liquidRaw'],
+			[/\%\}/, { token: 'delimiter.tag.liquid', switchTo: '@$S2.$S3' }],
 			{ include: 'liquidRoot' }
 		],
 
-		liquidInTagState: [
-			[/%\}/, { token: 'delimiter.output.liquid', switchTo: '@$S2.$S3' }],
-			// { include: 'liquidRoot' },
-			[/[^%]/, 'wut']
+		liquidRaw: [
+			[/^(?!\{\%\s*endraw\s*\%\}).+/],
+			[/\{\%/, 'delimiter.tag.liquid'],
+			[/@identifier/],
+			[/\%\}/, { token: 'delimiter.tag.liquid', next: '@root' }],
 		],
 
 		liquidRoot: [
 			[/\d+(\.\d+)?/, 'number.liquid'],
 			[/"[^"]*"/, 'string.liquid'],
 			[/'[^']*'/, 'string.liquid'],
-			[/[\s]+/],
+			[/\s+/],
+			[
+				/@symbol/,
+				{
+					cases: {
+						'@operators': 'operator.liquid',
+						'@default': ''
+					}
+				}
+			],
+			[/\./],
+			[
+				/@identifier/,
+				{
+					cases: {
+						'@constants': 'keyword.liquid',
+						'@builtinFilters': 'predefined.liquid',
+						'@builtinTags': 'predefined.liquid',
+						'@default': 'variable.liquid'
+					}
+				}
+			],
 			[/[^}|%]/, 'variable.liquid']
 		]
 	}