Generate classic-style tokens for bat

src/bat.ts

@@ -38,9 +38,9 @@ export var language = <ILanguage> {
 	tokenPostfix: '.bat',
 
 	brackets: [
-		{ token: 'punctuation.bracket', open: '{', close: '}' },
-		{ token: 'punctuation.parenthesis', open: '(', close: ')' },
-		{ token: 'punctuation.square', open: '[', close: ']' }
+		{ token: 'delimiter.bracket', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' }
 	],
 
 	keywords: /call|defined|echo|errorlevel|exist|for|goto|if|pause|set|shift|start|title|not|pushd|popd/,
@@ -55,14 +55,14 @@ export var language = <ILanguage> {
 
 			[/^(\s*)(rem(?:\s.*|))$/, ['','comment']],
 
-			[/(\@?)(@keywords)(?!\w)/, [{token:'support.function'}, {token:'support.function.$2'}]],
+			[/(\@?)(@keywords)(?!\w)/, [{token:'keyword'}, {token:'keyword.$2'}]],
 
 			// whitespace
 			[/[ \t\r\n]+/, ''],
 
 			// blocks
-			[/setlocal(?!\w)/, { token: 'support.function.tag-setlocal', bracket: '@open' }],
-			[/endlocal(?!\w)/, { token: 'support.function.tag-setlocal', bracket: '@close' }],
+			[/setlocal(?!\w)/, { token: 'keyword.tag-setlocal', bracket: '@open' }],
+			[/endlocal(?!\w)/, { token: 'keyword.tag-setlocal', bracket: '@close' }],
 
 			// words
 			[/[a-zA-Z_]\w*/, ''],
@@ -76,15 +76,15 @@ export var language = <ILanguage> {
 
 			// punctuations
 			[/[{}()\[\]]/, '@brackets'],
-			[/@symbols/, 'punctuation'],
+			[/@symbols/, 'delimiter'],
 
 			// numbers
-			[/\d*\.\d+([eE][\-+]?\d+)?/, 'constant.numeric.float'],
-			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'constant.numeric.hex'],
-			[/\d+/, 'constant.numeric'],
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+/, 'number'],
 
 			// punctuation: after number because of .\d floats
-			[/[;,.]/, 'punctuation'],
+			[/[;,.]/, 'delimiter'],
 
 			// strings:
 			[/"/,  'string', '@string."' ],

test/bat.test.ts

@@ -7,14 +7,14 @@
 import {testTokenization} from './testRunner';
 
 testTokenization('bat', [
-	// support.functions
+	// Keywords
 	[{
 	line: '@echo off title Selfhost',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.bat' },
-		{ startIndex: 1, type: 'support.function.echo.bat' },
+		{ startIndex: 0, type: 'keyword.bat' },
+		{ startIndex: 1, type: 'keyword.echo.bat' },
 		{ startIndex: 5, type: '' },
-		{ startIndex: 10, type: 'support.function.title.bat' },
+		{ startIndex: 10, type: 'keyword.title.bat' },
 		{ startIndex: 15, type: '' }
 	]}],
 
@@ -44,82 +44,82 @@ testTokenization('bat', [
 		{ startIndex: 0, type: '' }
 	]}],
 
-	// constant.numerics
+	// number
 	[{
 	line: '0',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.bat' }
+		{ startIndex: 0, type: 'number.bat' }
 	]}],
 
 	[{
 	line: '0.0',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+		{ startIndex: 0, type: 'number.float.bat' }
 	]}],
 
 	[{
 	line: '0x123',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.hex.bat' }
+		{ startIndex: 0, type: 'number.hex.bat' }
 	]}],
 
 	[{
 	line: '23.5',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+		{ startIndex: 0, type: 'number.float.bat' }
 	]}],
 
 	[{
 	line: '23.5e3',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+		{ startIndex: 0, type: 'number.float.bat' }
 	]}],
 
 	[{
 	line: '23.5E3',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+		{ startIndex: 0, type: 'number.float.bat' }
 	]}],
 
 	[{
 	line: '1.72e-3',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+		{ startIndex: 0, type: 'number.float.bat' }
 	]}],
 
 	[{
 	line: '0+0',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.bat' },
-		{ startIndex: 1, type: 'punctuation.bat' },
-		{ startIndex: 2, type: 'constant.numeric.bat' }
+		{ startIndex: 0, type: 'number.bat' },
+		{ startIndex: 1, type: 'delimiter.bat' },
+		{ startIndex: 2, type: 'number.bat' }
 	]}],
 
 	[{
 	line: '100+10',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.bat' },
-		{ startIndex: 3, type: 'punctuation.bat' },
-		{ startIndex: 4, type: 'constant.numeric.bat' }
+		{ startIndex: 0, type: 'number.bat' },
+		{ startIndex: 3, type: 'delimiter.bat' },
+		{ startIndex: 4, type: 'number.bat' }
 	]}],
 
 	[{
 	line: '0 + 0',
 	tokens: [
-		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 0, type: 'number.bat' },
 		{ startIndex: 1, type: '' },
-		{ startIndex: 2, type: 'punctuation.bat' },
+		{ startIndex: 2, type: 'delimiter.bat' },
 		{ startIndex: 3, type: '' },
-		{ startIndex: 4, type: 'constant.numeric.bat' }
+		{ startIndex: 4, type: 'number.bat' }
 	]}],
 
 	// Strings
 	[{
 	line: 'set s = "string"',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.set.bat' },
+		{ startIndex: 0, type: 'keyword.set.bat' },
 		{ startIndex: 3, type: '' },
-		{ startIndex: 6, type: 'punctuation.bat' },
+		{ startIndex: 6, type: 'delimiter.bat' },
 		{ startIndex: 7, type: '' },
 		{ startIndex: 8, type: 'string.bat' }
 	]}],
@@ -128,42 +128,42 @@ testTokenization('bat', [
 	line: '"use strict";',
 	tokens: [
 		{ startIndex: 0, type: 'string.bat' },
-		{ startIndex: 12, type: 'punctuation.bat' }
+		{ startIndex: 12, type: 'delimiter.bat' }
 	]}],
 
 	// Tags
 	[{
 	line: 'setlocal endlocal',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
 		{ startIndex: 8, type: '' },
-		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+		{ startIndex: 9, type: 'keyword.tag-setlocal.bat' }
 	]}],
 
 	[{
 	line: 'setlocal ENDLOCAL',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
 		{ startIndex: 8, type: '' },
-		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+		{ startIndex: 9, type: 'keyword.tag-setlocal.bat' }
 	]}],
 
 	[{
 	line: 'SETLOCAL endlocal',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
 		{ startIndex: 8, type: '' },
-		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+		{ startIndex: 9, type: 'keyword.tag-setlocal.bat' }
 	]}],
 
 	[{
 	line: 'setlocal setlocal endlocal',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
 		{ startIndex: 8, type: '' },
-		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 9, type: 'keyword.tag-setlocal.bat' },
 		{ startIndex: 17, type: '' },
-		{ startIndex: 18, type: 'support.function.tag-setlocal.bat' }
+		{ startIndex: 18, type: 'keyword.tag-setlocal.bat' }
 	]}],
 
 	// Monarch generated
@@ -187,7 +187,7 @@ testTokenization('bat', [
 	line: 'REMOVED not a comment really',
 	tokens: [
 		{ startIndex: 0, type: '' },
-		{ startIndex: 8, type: 'support.function.not.bat' },
+		{ startIndex: 8, type: 'keyword.not.bat' },
 		{ startIndex: 11, type: '' }
 	]}, {
 	line: '',
@@ -196,13 +196,13 @@ testTokenization('bat', [
 	]}, {
 	line: 'echo cool',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.echo.bat' },
+		{ startIndex: 0, type: 'keyword.echo.bat' },
 		{ startIndex: 4, type: '' }
 	]}, {
 	line: '@echo off',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.bat' },
-		{ startIndex: 1, type: 'support.function.echo.bat' },
+		{ startIndex: 0, type: 'keyword.bat' },
+		{ startIndex: 1, type: 'keyword.echo.bat' },
 		{ startIndex: 5, type: '' }
 	]}, {
 	line: '',
@@ -211,7 +211,7 @@ testTokenization('bat', [
 	]}, {
 	line: 'setlocAL',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' }
+		{ startIndex: 0, type: 'keyword.tag-setlocal.bat' }
 	]}, {
 	line: '	asdf',
 	tokens: [
@@ -223,7 +223,7 @@ testTokenization('bat', [
 	]}, {
 	line: 'endLocaL',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' }
+		{ startIndex: 0, type: 'keyword.tag-setlocal.bat' }
 	]}, {
 	line: '',
 	tokens: [
@@ -231,7 +231,7 @@ testTokenization('bat', [
 	]}, {
 	line: 'call',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.call.bat' }
+		{ startIndex: 0, type: 'keyword.call.bat' }
 	]}, {
 	line: '',
 	tokens: [
@@ -275,34 +275,34 @@ testTokenization('bat', [
 	]}, {
 	line: 'FOR %%A IN (1 2 3) DO (',
 	tokens: [
-		{ startIndex: 0, type: 'support.function.for.bat' },
+		{ startIndex: 0, type: 'keyword.for.bat' },
 		{ startIndex: 3, type: '' },
 		{ startIndex: 4, type: 'variable.bat' },
 		{ startIndex: 7, type: '' },
-		{ startIndex: 11, type: 'punctuation.parenthesis.bat' },
-		{ startIndex: 12, type: 'constant.numeric.bat' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.bat' },
+		{ startIndex: 12, type: 'number.bat' },
 		{ startIndex: 13, type: '' },
-		{ startIndex: 14, type: 'constant.numeric.bat' },
+		{ startIndex: 14, type: 'number.bat' },
 		{ startIndex: 15, type: '' },
-		{ startIndex: 16, type: 'constant.numeric.bat' },
-		{ startIndex: 17, type: 'punctuation.parenthesis.bat' },
+		{ startIndex: 16, type: 'number.bat' },
+		{ startIndex: 17, type: 'delimiter.parenthesis.bat' },
 		{ startIndex: 18, type: '' },
-		{ startIndex: 22, type: 'punctuation.parenthesis.bat' }
+		{ startIndex: 22, type: 'delimiter.parenthesis.bat' }
 	]}, {
 	line: '	SET VAR1=%VAR1%%%A',
 	tokens: [
 		{ startIndex: 0, type: '' },
-		{ startIndex: 1, type: 'support.function.set.bat' },
+		{ startIndex: 1, type: 'keyword.set.bat' },
 		{ startIndex: 4, type: '' },
-		{ startIndex: 9, type: 'punctuation.bat' },
+		{ startIndex: 9, type: 'delimiter.bat' },
 		{ startIndex: 10, type: 'variable.bat' }
 	]}, {
 	line: '	SET VAR2=%VAR2%%%A',
 	tokens: [
 		{ startIndex: 0, type: '' },
-		{ startIndex: 1, type: 'support.function.set.bat' },
+		{ startIndex: 1, type: 'keyword.set.bat' },
 		{ startIndex: 4, type: '' },
-		{ startIndex: 9, type: 'punctuation.bat' },
+		{ startIndex: 9, type: 'delimiter.bat' },
 		{ startIndex: 10, type: 'variable.bat' }
 	]}, {
 	line: '	use \'string %%a asdf asdf\'',
@@ -322,11 +322,11 @@ testTokenization('bat', [
 	line: '	this shold NOT BE red',
 	tokens: [
 		{ startIndex: 0, type: '' },
-		{ startIndex: 12, type: 'support.function.not.bat' },
+		{ startIndex: 12, type: 'keyword.not.bat' },
 		{ startIndex: 15, type: '' }
 	]}, {
 	line: ')',
 	tokens: [
-		{ startIndex: 0, type: 'punctuation.parenthesis.bat' }
+		{ startIndex: 0, type: 'delimiter.parenthesis.bat' }
 	]}]
 ]);