Merge remote-tracking branch 'origin/master' into pr/Vityanchys/43

.editorconfig

@@ -0,0 +1,16 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Tab indentation
+[*]
+indent_style = tab
+indent_size = 4
+trim_trailing_whitespace = true
+
+# The indent size used in the `package.json` file cannot be changed
+# https://github.com/npm/npm/pull/3180#issuecomment-16336516
+[{*.yml,*.yaml,package.json}]
+indent_style = space
+indent_size = 2

.vscode/launch.json

@@ -8,8 +8,8 @@
             "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
             "stopOnEntry": false,
             "args": [
-                "--grep",
-                "php"
+                // "--grep",
+                // "typescript"
             ],
             "cwd": "${workspaceRoot}",
             "preLaunchTask": null,
@@ -25,4 +25,4 @@
             "outDir": null
         }
     ]
-}
+}

.vscode/settings.json

@@ -9,5 +9,6 @@
 	"files.insertFinalNewline": true,
 	"editor.tabSize": 4,
 	"editor.insertSpaces": false,
+	"editor.detectIndentation": false,
 	"typescript.tsdk": "./node_modules/typescript/lib"
 }

README.md

@@ -18,6 +18,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed
 * html
 * ini
 * java
+* javascript
 * lua
 * markdown
 * msdax
@@ -39,6 +40,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed
 * sql
 * st
 * swift
+* typescript
 * vb
 * xml
 * yaml

scripts/bundle.js

@@ -34,6 +34,7 @@ bundleOne('html/html');
 bundleOne('ini/ini');
 bundleOne('pug/pug');
 bundleOne('java/java');
+bundleOne('javascript/javascript');
 bundleOne('less/less');
 bundleOne('lua/lua');
 bundleOne('markdown/markdown');
@@ -51,6 +52,7 @@ bundleOne('scss/scss');
 bundleOne('sql/sql');
 bundleOne('st/st');
 bundleOne('swift/swift');
+bundleOne('typescript/typescript');
 bundleOne('vb/vb');
 bundleOne('xml/xml');
 bundleOne('yaml/yaml');
@@ -65,6 +67,7 @@ bundleOne('scheme/scheme');
 bundleOne('clojure/clojure');
 bundleOne('shell/shell');
 bundleOne('perl/perl'),
+bundleOne('powerquery/powerquery')
 bundleOne('azcli/azcli')
 
 function bundleOne(moduleId, exclude) {

src/clojure/clojure.contribution.ts

@@ -8,11 +8,11 @@ import { registerLanguage } from '../_.contribution';
 
 // Allow for running under nodejs/requirejs in tests
 const _monaco: typeof monaco =
-    typeof monaco === 'undefined' ? (<any>self).monaco : monaco;
+	typeof monaco === 'undefined' ? (<any>self).monaco : monaco;
 
 registerLanguage({
-    id: 'clojure',
-    extensions: ['.clj', '.clojure'],
-    aliases: ['clojure', 'Clojure'],
-    loader: () => _monaco.Promise.wrap(import('./clojure')),
+	id: 'clojure',
+	extensions: ['.clj', '.clojure'],
+	aliases: ['clojure', 'Clojure'],
+	loader: () => _monaco.Promise.wrap(import('./clojure')),
 });

src/clojure/clojure.test.ts

@@ -8,97 +8,83 @@
 import { testTokenization } from '../test/testRunner';
 
 testTokenization('clojure', [
-  // Keywords
-  [
-    {
-      line: 'defmacro some',
-      tokens: [
-        { startIndex: 0, type: 'keyword.clj' },
-        { startIndex: 8, type: 'white.clj' },
-        { startIndex: 9, type: 'variable.clj' },
-      ],
-    },
+	// Keywords
+	[
+		{
+			line: 'defmacro some',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.clj' },
+				{ startIndex: 8, type: 'white.clj' },
+				{ startIndex: 9, type: 'variable.clj' },
+			],
+		},
 
-    {
-      line: 'comment "text comment"',
-      tokens: [
-	{ startIndex: 0, type: 'keyword.clj' },
-	{ startIndex: 7, type: 'white.clj'},
-	{ startIndex: 8, type: 'string.clj'},
-      ],
-    },
-    {
-      line: 'in-ns "user',
-      tokens: [
-        { startIndex: 0, type: 'keyword.clj' },
-        { startIndex: 5, type: 'white.clj' },
-        { startIndex: 6, type: 'string.clj' },
-      ],
-    },
-  ],
+		{
+			line: 'comment "text comment"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.clj' },
+				{ startIndex: 7, type: 'white.clj' },
+				{ startIndex: 8, type: 'string.clj' },
+			],
+		},
+		{
+			line: 'in-ns "user',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.clj' },
+				{ startIndex: 5, type: 'white.clj' },
+				{ startIndex: 6, type: 'string.clj' },
+			],
+		},
+	],
 
-  // comments
-  [
-    {
-      line: ';; comment',
-      tokens: [{ startIndex: 0, type: 'comment.clj' }],
-    },
-  ],
-  [
-    {
-      line: '(comment',
-      tokens: [{ startIndex: 0, type: 'comment.clj' }],
-    },
-    {
-      line: '(comment let',
-      tokens: [
-        { startIndex: 0, type: 'comment.clj' },
-        { startIndex: 8, type: 'white.clj' },
-        { startIndex: 9, type: 'keyword.clj' },
-      ],
-    },
-  ],
+	// comments
+	[
+		{
+			line: ';; comment',
+			tokens: [{ startIndex: 0, type: 'comment.clj' }],
+		},
+	],
 
-  // strings
-  [
-    {
-      line: '"\\n string "',
-      tokens: [
-        { startIndex: 0, type: 'string.clj' },
-        { startIndex: 1, type: 'string.escape.clj' },
-        { startIndex: 3, type: 'string.clj' },
-      ],
-    },
-  ],
-  [
-    {
-      line: '" string \\',
-      tokens: [{ startIndex: 0, type: 'string.clj' }],
-    },
-    {
-      line: 'multiline',
-      tokens: [{ startIndex: 0, type: 'string.clj' }],
-    },
-    {
-      line: ' ',
-      tokens: [
-        // previous line needs to be terminated with \
-        { startIndex: 0, type: 'white.clj' },
-      ],
-    },
-  ],
+	// strings
+	[
+		{
+			line: '"\\n string "',
+			tokens: [
+				{ startIndex: 0, type: 'string.clj' },
+				{ startIndex: 1, type: 'string.escape.clj' },
+				{ startIndex: 3, type: 'string.clj' },
+			],
+		},
+	],
+	[
+		{
+			line: '" string \\',
+			tokens: [{ startIndex: 0, type: 'string.clj' }],
+		},
+		{
+			line: 'multiline',
+			tokens: [{ startIndex: 0, type: 'string.clj' }],
+		},
+		{
+			line: ' ',
+			tokens: [
+				// previous line needs to be terminated with \
+				{ startIndex: 0, type: 'white.clj' },
+			],
+		},
+	],
 
-  // numbers
-  [
-    {
-      line: '1e2',
-      tokens: [{ startIndex: 0, type: 'number.float.clj' }],
-    },
-  ],
-  [
-    {
-      line: '#x03BB',
-      tokens: [{ startIndex: 0, type: 'number.hex.clj' }],
-    },
-  ],
+	// numbers
+	[
+		{
+			line: '1e2',
+			tokens: [{ startIndex: 0, type: 'number.float.clj' }],
+		},
+	],
+	[
+		{
+			line: '0xff',
+			tokens: [{ startIndex: 0, type: 'number.hex.clj' }],
+		},
+	],
 ]);

src/clojure/clojure.ts

@@ -9,205 +9,219 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
 export const conf: IRichLanguageConfiguration = {
-  comments: {
-    lineComment: ';;',
-    blockComment: ['(comment', ')'],
-  },
-
-  brackets: [['(', ')'], ['{', '}'], ['[', ']']],
-
-  autoClosingPairs: [
-    { open: '{', close: '}' },
-    { open: '[', close: ']' },
-    { open: '(', close: ')' },
-    { open: '"', close: '"' },
-  ],
-
-  surroundingPairs: [
-    { open: '{', close: '}' },
-    { open: '[', close: ']' },
-    { open: '(', close: ')' },
-    { open: '"', close: '"' },
-  ],
+	comments: {
+		lineComment: ';;',
+	},
+
+	brackets: [['(', ')'], ['{', '}'], ['[', ']']],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+	],
+
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+	],
 };
 
 export const language = <ILanguage>{
-  defaultToken: '',
-  ignoreCase: true,
-  tokenPostfix: '.clj',
-
-  brackets: [
-    { open: '(', close: ')', token: 'delimiter.parenthesis' },
-    { open: '{', close: '}', token: 'delimiter.curly' },
-    { open: '[', close: ']', token: 'delimiter.square' },
-  ],
-
-  keywords: [
-    'ns',
-    'ns\-unmap',
-    'create\-ns',
-    'in\-ns',
-    'fn',
-    'def',
-    'defn',
-    'defmacro',
-    'defmulti',
-    'defonce',
-    'require',
-    'import',
-    'new',
-    'refer',
-    'pos',
-    'pos\?',
-    'filter',
-    'map',
-    'reduce',
-    'repeat',
-    'key',
-    'rest',
-    'concat',
-    'into',
-    'reverse',
-    'iterate',
-    'range',
-    'drop',
-    'drop\-while',
-    'take',
-    'take\-while',
-    'neg',
-    'neg?',
-    'bound\-fn',
-    'if',
-    'if\-not',
-    'if\-let',
-    'case,',
-    'contains',
-    'conj',
-    'disj',
-    'sort',
-    'get',
-    'assoc',
-    'merge',
-    'keys',
-    'vals',
-    'nth',
-    'first',
-    'last',
-    'count',
-    'contains?',
-    'cond',
-    'condp',
-    'cond\-\>',
-    'cond\-\>\>',
-    'when',
-    'while',
-    'when\-not',
-    'when\-let',
-    'when\-first',
-    'do',
-    'future',
-    'comment',
-    'doto',
-    'locking',
-    'proxy',
-    'println',
-    'type',
-    'meta',
-    'var',
-    'as\-\>',
-    'reify',
-    'deftype',
-    'defrecord',
-    'defprotocol',
-    'extend',
-    'extend-protocol',
-    'extend-type',
-    'specify',
-    'specify\!',
-    'try',
-    'catch',
-    'finally',
-    'let',
-    'letfn',
-    'binding',
-    'loop',
-    'for',
-    'seq',
-    'doseq',
-    'dotimes',
-    'when\-let',
-    'if\-let',
-    'when\-some',
-    'if\-some',
-    'this\-as',
-    'defmethod',
-    'testing',
-    'deftest',
-    'are',
-    'use\-fixtures',
-    'use',
-    'remove',
-    'run',
-    'run\*',
-    'fresh',
-    'alt!',
-    'alt!!',
-    'go',
-    'go\-loop',
-    'thread',
-  ],
-
-  constants: ['true', 'false', 'nil', 'boolean', 'str'],
-
-  operators: ['=', 'not=', '<', '<=', '>', '>=', 'and', 'or', 'not', 'inc', 'dec', 'max', 'min', 'rem', 'bit-and', 'bit-or', 'bit-xor', 'bit-not'],
-
-  tokenizer: {
-    root: [
-      [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'],
-      [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'],
-
-      [/(?:\b(?:(def|defn|defmacro|defmulti|defonce|ns|ns-unmap|fn))\b)(\s+)((?:\w|\-|\!|\?)*)/, ['keyword', 'white', 'variable']],
-
-      [
-        /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/,
-        {
-          cases: {
-            '@keywords': 'keyword',
-            '@constants': 'constant',
-            '@operators': 'operators',
-            '@default': 'identifier',
-          },
-        },
-      ],
-
-      { include: '@whitespace' },
-      { include: '@strings' },
-    ],
-
-    comment: [
-      [/[^\(comment]+/, 'comment'],
-      [/\)/, 'comment', '@push'],
-      [/\(comment/, 'comment', '@pop'],
-      [/[\)]/, 'comment'],
-    ],
-
-    whitespace: [
-      [/[ \t\r\n]+/, 'white'],
-      [/\(comment/, 'comment', '@comment'],
-      [/;;.*$/, 'comment'],
-    ],
-
-    strings: [
-      [/"$/, 'string', '@popall'],
-      [/"(?=.)/, 'string', '@multiLineString'],
-    ],
-
-    multiLineString: [
-      [/\\./, 'string.escape'],
-      [/"/, 'string', '@popall'],
-      [/.(?=.*")/, 'string'],
-      [/.*\\$/, 'string'],
-      [/.*$/, 'string', '@popall'],
-    ],
-  },
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.clj',
+
+	brackets: [
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+	],
+
+	keywords: [
+		'ns',
+		'ns-unmap',
+		'create-ns',
+		'in-ns',
+		'fn',
+		'def',
+		'defn',
+		'defmacro',
+		'defmulti',
+		'defonce',
+		'require',
+		'import',
+		'new',
+		'refer',
+		'pos',
+		'pos?',
+		'filter',
+		'map',
+		'reduce',
+		'repeat',
+		'key',
+		'rest',
+		'concat',
+		'into',
+		'reverse',
+		'iterate',
+		'range',
+		'drop',
+		'drop-while',
+		'take',
+		'take-while',
+		'neg',
+		'neg?',
+		'bound-fn',
+		'if',
+		'if-not',
+		'if-let',
+		'case,',
+		'contains',
+		'conj',
+		'disj',
+		'sort',
+		'get',
+		'assoc',
+		'merge',
+		'keys',
+		'vals',
+		'nth',
+		'first',
+		'last',
+		'count',
+		'contains?',
+		'cond',
+		'condp',
+		'cond->',
+		'cond->>',
+		'when',
+		'while',
+		'when-not',
+		'when-let',
+		'when-first',
+		'do',
+		'future',
+		'comment',
+		'doto',
+		'locking',
+		'proxy',
+		'println',
+		'type',
+		'meta',
+		'var',
+		'as->',
+		'reify',
+		'deftype',
+		'defrecord',
+		'defprotocol',
+		'extend',
+		'extend-protocol',
+		'extend-type',
+		'specify',
+		'specify!',
+		'try',
+		'catch',
+		'finally',
+		'let',
+		'letfn',
+		'binding',
+		'loop',
+		'for',
+		'seq',
+		'doseq',
+		'dotimes',
+		'when-let',
+		'if-let',
+		'when-some',
+		'if-some',
+		'this-as',
+		'defmethod',
+		'testing',
+		'deftest',
+		'are',
+		'use-fixtures',
+		'use',
+		'remove',
+		'run',
+		'run*',
+		'fresh',
+		'alt!',
+		'alt!!',
+		'go',
+		'go-loop',
+		'thread',
+		'boolean',
+		'str',
+	],
+
+	constants: ['true', 'false', 'nil'],
+
+	operators: [
+		'=',
+		'not=',
+		'<',
+		'<=',
+		'>',
+		'>=',
+		'and',
+		'or',
+		'not',
+		'inc',
+		'dec',
+		'max',
+		'min',
+		'rem',
+		'bit-and',
+		'bit-or',
+		'bit-xor',
+		'bit-not',
+	],
+
+	tokenizer: {
+		root: [
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'],
+
+			[
+				/(?:\b(?:(ns|def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))(?![\w-]))(\s+)((?:\w|\-|\!|\?)*)/,
+				['keyword', 'white', 'variable'],
+			],
+
+			[
+				/[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@constants': 'constant',
+						'@operators': 'operators',
+						'@default': 'identifier',
+					},
+				},
+			],
+
+			[/\/#"(?:\.|(?:\")|[^""\n])*"\/g/, 'regexp'],
+
+			{ include: '@whitespace' },
+			{ include: '@strings' },
+		],
+
+		whitespace: [[/[ \t\r\n]+/, 'white'], [/;;.*$/, 'comment']],
+
+		strings: [
+			[/"$/, 'string', '@popall'],
+			[/"(?=.)/, 'string', '@multiLineString'],
+		],
+
+		multiLineString: [
+			[/\\./, 'string.escape'],
+			[/"/, 'string', '@popall'],
+			[/.(?=.*")/, 'string'],
+			[/.*\\$/, 'string'],
+			[/.*$/, 'string', '@popall'],
+		],
+	},
 };

src/html/html.test.ts

@@ -7,7 +7,7 @@
 
 import { testTokenization } from '../test/testRunner';
 
-testTokenization(['html', 'css'], [
+testTokenization(['html', 'css', 'javascript'], [
 
 	// Open Start Tag #1'
 	[{
@@ -147,7 +147,13 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: 'attribute.value.html' },
 			{ startIndex: 30, type: 'delimiter.html' },
-			{ startIndex: 31, type: '' },
+			{ startIndex: 31, type: 'keyword.js' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'identifier.js' },
+			{ startIndex: 36, type: 'delimiter.js' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'number.js' },
+			{ startIndex: 40, type: 'delimiter.js' },
 			{ startIndex: 41, type: 'delimiter.html' },
 			{ startIndex: 43, type: 'tag.html' },
 			{ startIndex: 49, type: 'delimiter.html' }
@@ -169,7 +175,13 @@ testTokenization(['html', 'css'], [
 	}, {
 		line: 'var i= 10;',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
 		]
 	}, {
 		line: '</script>',
@@ -191,7 +203,14 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: 'attribute.value.html' },
 			{ startIndex: 30, type: 'delimiter.html' },
-			{ startIndex: 31, type: '' },
+			{ startIndex: 31, type: 'keyword.js' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'identifier.js' },
+			{ startIndex: 36, type: 'delimiter.js' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'number.js' },
+			{ startIndex: 40, type: 'delimiter.js' },
+
 		]
 	}, {
 		line: '</script>',
@@ -217,7 +236,13 @@ testTokenization(['html', 'css'], [
 	}, {
 		line: 'var i= 10;</script>',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
 			{ startIndex: 10, type: 'delimiter.html' },
 			{ startIndex: 12, type: 'tag.html' },
 			{ startIndex: 18, type: 'delimiter.html' }
@@ -254,14 +279,14 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
 			{ startIndex: 9, type: 'delimiter.html' },
 			{ startIndex: 11, type: 'tag.html' },
 			{ startIndex: 17, type: 'delimiter.html' },
 			// { startIndex:18, type: 'delimiter.html' },
 			{ startIndex: 19, type: 'tag.html' },
 			{ startIndex: 25, type: 'delimiter.html' },
-			{ startIndex: 26, type: '' },
+			{ startIndex: 26, type: 'identifier.js' },
 			{ startIndex: 27, type: 'delimiter.html' },
 			{ startIndex: 29, type: 'tag.html' },
 			{ startIndex: 35, type: 'delimiter.html' }
@@ -292,7 +317,13 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.js' },
+			{ startIndex: 13, type: 'delimiter.js' },
+			{ startIndex: 14, type: '' },
+			{ startIndex: 15, type: 'number.js' },
+			{ startIndex: 17, type: 'delimiter.js' },
 			{ startIndex: 18, type: 'delimiter.html' },
 			{ startIndex: 20, type: 'tag.html' },
 			{ startIndex: 26, type: 'delimiter.html' }

src/javascript/javascript.contribution.ts

@@ -0,0 +1,20 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+registerLanguage({
+	id: 'javascript',
+	extensions: ['.js', '.es6', '.jsx'],
+	firstLine: '^#!.*\\bnode',
+	filenames: ['jakefile'],
+	aliases: ['JavaScript', 'javascript', 'js'],
+	mimetypes: ['text/javascript'],
+	loader: () => _monaco.Promise.wrap(import('./javascript'))
+});

src/javascript/javascript.test.ts

@@ -0,0 +1,553 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('javascript', [
+	// Keywords
+	[{
+		line: 'var x = function() { };',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 16, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 18, type: '' },
+			{ startIndex: 19, type: 'delimiter.bracket.js' },
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'delimiter.bracket.js' },
+			{ startIndex: 22, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '    var    ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'keyword.js' },
+			{ startIndex: 7, type: '' }
+		]
+	}],
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '// a comment /*',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '// a comment /**',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: 'var x = 1; // my comment // is a nice one',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'comment.js' }
+		]
+	}],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.js' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.js' },
+			{ startIndex: 32, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.js' },
+			{ startIndex: 8, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'x = /*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.js' }
+		]
+	}],
+
+	// Comments - range comment, multi lines
+	[{
+		line: '/* a multiline comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}, {
+		line: 'can actually span',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}, {
+		line: 'multiple lines */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* start a comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.js' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}, {
+		line: 'and end it */ var a = 2;',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.js' },
+			{ startIndex: 17, type: '' },
+			{ startIndex: 18, type: 'identifier.js' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'delimiter.js' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.js' },
+			{ startIndex: 23, type: 'delimiter.js' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: 'var a = \'a\';',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.js' },
+			{ startIndex: 11, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '"use strict";',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 12, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'b = a + " \'cool\'  "',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.js' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 10, type: 'string.escape.js' },
+			{ startIndex: 12, type: 'string.js' },
+			{ startIndex: 18, type: 'string.escape.js' },
+			{ startIndex: 20, type: 'string.js' },
+		]
+	}],
+
+	[{
+		line: '\'\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 2, type: 'string.invalid.js' },
+		]
+	}],
+
+	[{
+		line: '\'\\\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 1, type: 'string.escape.js' },
+			{ startIndex: 3, type: 'string.js' },
+		]
+	}],
+
+	[{
+		line: '\'be careful \\not to escape\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 12, type: 'string.escape.js' },
+			{ startIndex: 14, type: 'string.js' },
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: ' 0',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: ' 0 ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.js' },
+			{ startIndex: 2, type: '' }
+		]
+	}],
+
+	[{
+		line: '0 ',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: 'delimiter.js' },
+			{ startIndex: 2, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 3, type: 'delimiter.js' },
+			{ startIndex: 4, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '0123',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.js' }
+		]
+	}],
+
+	[{
+		line: '01239',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.js' },
+			{ startIndex: 4, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: 'identifier.js' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.js' }
+		]
+	}],
+
+	// Regular Expressions
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '/**/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '/***/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.js' }
+		]
+	}],
+
+	[{
+		line: '5 / 3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: 'delimiter.js' }
+		]
+	}],
+
+	// Advanced regular expressions
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '1 / 2 / x / b;',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'delimiter.js' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.js' },
+			{ startIndex: 13, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'a /ads/ b;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: 'identifier.js' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
+			{ startIndex: 9, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '1/(2/3)/2/3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: 'delimiter.js' },
+			{ startIndex: 2, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 3, type: 'number.js' },
+			{ startIndex: 4, type: 'delimiter.js' },
+			{ startIndex: 5, type: 'number.js' },
+			{ startIndex: 6, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 7, type: 'delimiter.js' },
+			{ startIndex: 8, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+			{ startIndex: 10, type: 'number.js' },
+			{ startIndex: 11, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '{ key: 123 }',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.bracket.js' }
+		]
+	}],
+
+	[{
+		line: '[1,2,3]',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.square.js' },
+			{ startIndex: 1, type: 'number.js' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: 'number.js' },
+			{ startIndex: 4, type: 'delimiter.js' },
+			{ startIndex: 5, type: 'number.js' },
+			{ startIndex: 6, type: 'delimiter.square.js' }
+		]
+	}],
+
+	[{
+		line: 'foo(123);',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 3, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 7, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 8, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '{a:{b:[]}}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.js' },
+			{ startIndex: 1, type: 'identifier.js' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: 'delimiter.bracket.js' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: 'delimiter.square.js' },
+			{ startIndex: 8, type: 'delimiter.bracket.js' }
+		]
+	}],
+
+	[{
+		line: 'x = "[{()}]"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'string.js' }
+		]
+	}],
+
+
+	[{
+		line: '`${5 + \'x\' + 3}a${4}`',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 1, type: 'delimiter.bracket.js' },
+			{ startIndex: 3, type: 'number.js' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'string.js' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.js' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'number.js' },
+			{ startIndex: 14, type: 'delimiter.bracket.js' },
+			{ startIndex: 15, type: 'string.js' },
+			{ startIndex: 16, type: 'delimiter.bracket.js' },
+			{ startIndex: 18, type: 'number.js' },
+			{ startIndex: 19, type: 'delimiter.bracket.js' },
+			{ startIndex: 20, type: 'string.js' },
+
+		]
+	}]
+
+]);

src/javascript/javascript.ts

@@ -0,0 +1,41 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { conf as tsConf, language as tsLanguage } from '../typescript/typescript';
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+export const conf: IRichLanguageConfiguration = tsConf;
+
+export const language = <ILanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.js',
+
+	keywords: [
+		'break', 'case', 'catch', 'class', 'continue', 'const',
+		'constructor', 'debugger', 'default', 'delete', 'do', 'else',
+		'export', 'extends', 'false', 'finally', 'for', 'from', 'function',
+		'get', 'if', 'import', 'in', 'instanceof', 'let', 'new', 'null',
+		'return', 'set', 'super', 'switch', 'symbol', 'this', 'throw', 'true',
+		'try', 'typeof', 'undefined', 'var', 'void', 'while', 'with', 'yield',
+		'async', 'await', 'of'
+	],
+	typeKeywords: [],
+
+	operators: tsLanguage.operators,
+	symbols: tsLanguage.symbols,
+	escapes: tsLanguage.escapes,
+	digits: tsLanguage.digits,
+	octaldigits: tsLanguage.octaldigits,
+	binarydigits: tsLanguage.binarydigits,
+	hexdigits: tsLanguage.hexdigits,
+	tokenizer: tsLanguage.tokenizer,
+};

src/less/less.test.ts

@@ -7,7 +7,7 @@
 
 import { testTokenization } from '../test/testRunner';
 
-testTokenization(['less'], [
+testTokenization(['less', 'javascript'], [
 
 	// Keywords
 	[{
@@ -641,7 +641,10 @@ testTokenization(['less'], [
 			{ startIndex: 5, type: 'delimiter.less' },
 			{ startIndex: 6, type: '' },
 			{ startIndex: 7, type: 'delimiter.backtick.less' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'identifier.js' },
+			{ startIndex: 24, type: 'delimiter.parenthesis.js' },
 			{ startIndex: 26, type: 'delimiter.backtick.less' },
 			{ startIndex: 27, type: '' },
 			{ startIndex: 28, type: 'comment.less' }

src/monaco.contribution.ts

@@ -17,6 +17,7 @@ import './handlebars/handlebars.contribution';
 import './html/html.contribution';
 import './ini/ini.contribution';
 import './java/java.contribution';
+import './javascript/javascript.contribution';
 import './less/less.contribution';
 import './lua/lua.contribution';
 import './markdown/markdown.contribution';
@@ -26,6 +27,7 @@ import './objective-c/objective-c.contribution';
 import './pgsql/pgsql.contribution';
 import './php/php.contribution';
 import './postiats/postiats.contribution';
+import './powerquery/powerquery.contribution';
 import './powershell/powershell.contribution';
 import './pug/pug.contribution';
 import './python/python.contribution';
@@ -41,6 +43,7 @@ import './solidity/solidity.contribution';
 import './sql/sql.contribution';
 import './st/st.contribution';
 import './swift/swift.contribution';
+import './typescript/typescript.contribution';
 import './vb/vb.contribution';
 import './xml/xml.contribution';
 import './yaml/yaml.contribution';

src/php/php.test.ts

@@ -7,7 +7,7 @@
 
 import { testTokenization } from '../test/testRunner';
 
-testTokenization(['php', 'css'], [
+testTokenization(['php', 'css', 'javascript'], [
 	// Bug 13596:[ErrorTelemetry] Stream did not advance while tokenizing. Mode id is php (stuck)
 	// We're testing the fact that tokenize does not throw
 	[
@@ -1863,7 +1863,13 @@ testTokenization(['php', 'css'], [
 			// { startIndex:5, type: 'delimiter.html' },
 			{ startIndex: 6, type: 'tag.html' },
 			{ startIndex: 12, type: 'delimiter.html' },
-			{ startIndex: 13, type: '' },
+			{ startIndex: 13, type: 'keyword.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'identifier.js' },
+			{ startIndex: 18, type: 'delimiter.js' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'number.js' },
+			{ startIndex: 22, type: 'delimiter.js' },
 			{ startIndex: 23, type: 'delimiter.html' },
 			{ startIndex: 25, type: 'tag.html' },
 			{ startIndex: 31, type: 'delimiter.html' },
@@ -1888,7 +1894,13 @@ testTokenization(['php', 'css'], [
 			// { startIndex:5, type: 'delimiter.html' },
 			{ startIndex: 6, type: 'tag.html' },
 			{ startIndex: 12, type: 'delimiter.html' },
-			{ startIndex: 13, type: '' },
+			{ startIndex: 13, type: 'keyword.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'identifier.js' },
+			{ startIndex: 18, type: 'delimiter.js' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'number.js' },
+			{ startIndex: 22, type: 'delimiter.js' },
 			{ startIndex: 23, type: 'delimiter.html' },
 			{ startIndex: 25, type: 'tag.html' },
 			{ startIndex: 31, type: 'delimiter.html' },
@@ -1900,7 +1912,13 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 42, type: 'delimiter.html' },
 			{ startIndex: 43, type: 'tag.html' },
 			{ startIndex: 49, type: 'delimiter.html' },
-			{ startIndex: 50, type: '' },
+			{ startIndex: 50, type: 'keyword.js' },
+			{ startIndex: 53, type: '' },
+			{ startIndex: 54, type: 'identifier.js' },
+			{ startIndex: 55, type: 'delimiter.js' },
+			{ startIndex: 56, type: '' },
+			{ startIndex: 57, type: 'number.js' },
+			{ startIndex: 59, type: 'delimiter.js' },
 			{ startIndex: 60, type: 'delimiter.html' },
 			{ startIndex: 62, type: 'tag.html' },
 			{ startIndex: 68, type: 'delimiter.html' }
@@ -1969,12 +1987,28 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 73, type: 'delimiter.html' },
 			{ startIndex: 74, type: 'tag.html' },
 			{ startIndex: 80, type: 'delimiter.html' },
-			{ startIndex: 81, type: '' },
+			{ startIndex: 81, type: 'keyword.js' },
+			{ startIndex: 84, type: '' },
+			{ startIndex: 85, type: 'identifier.js' },
+			{ startIndex: 86, type: '' },
+			{ startIndex: 87, type: 'delimiter.js' },
+			{ startIndex: 88, type: '' },
+			{ startIndex: 89, type: 'number.js' },
+			{ startIndex: 90, type: 'delimiter.js' },
+			{ startIndex: 91, type: 'comment.js' },
 			{ startIndex: 94, type: 'metatag.php' },
 			{ startIndex: 97, type: 'string.php' },
 			{ startIndex: 109, type: 'comment.php' },
 			{ startIndex: 122, type: 'metatag.php' },
 			{ startIndex: 124, type: '' },
+			{ startIndex: 127, type: 'keyword.js' },
+			{ startIndex: 130, type: '' },
+			{ startIndex: 131, type: 'identifier.js' },
+			{ startIndex: 132, type: '' },
+			{ startIndex: 133, type: 'delimiter.js' },
+			{ startIndex: 134, type: '' },
+			{ startIndex: 135, type: 'number.js' },
+			{ startIndex: 136, type: 'delimiter.js' },
 			{ startIndex: 137, type: 'delimiter.html' },
 			{ startIndex: 139, type: 'tag.html' },
 			{ startIndex: 145, type: 'delimiter.html' },
@@ -2010,7 +2044,7 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'comment.js' },
 			{ startIndex: 10, type: 'metatag.php' }
 		]
 	}],
@@ -2021,13 +2055,13 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'string.invalid.js' },
 			{ startIndex: 9, type: 'metatag.php' },
 			{ startIndex: 14, type: 'number.php' },
 			{ startIndex: 15, type: 'delimiter.php' },
 			{ startIndex: 16, type: 'number.php' },
 			{ startIndex: 17, type: 'metatag.php' },
-			{ startIndex: 19, type: '' }
+			{ startIndex: 19, type: 'string.invalid.js' }
 		]
 	}],
 

src/powerquery/powerquery.contribution.ts

@@ -0,0 +1,17 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+registerLanguage({
+	id: 'powerquery',
+	extensions: ['.pq', '.pqm'],
+	aliases: ['PQ', 'M', 'Power Query', 'Power Query M'],
+	loader: () => _monaco.Promise.wrap(import('./powerquery'))
+});

src/powerquery/powerquery.test.ts

@@ -0,0 +1,374 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('powerquery', [
+	// Comments
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '    // a comment */',
+		tokens: [
+			{ startIndex: 0, type: 'white.pq' },
+			{ startIndex: 4, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '// /* #A */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '/*ABCD12$!()\\u000D%%%%%*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '42 /* + 45 */ /*',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' },
+			{ startIndex: 2, type: 'white.pq' },
+			{ startIndex: 3, type: 'comment.pq' },
+			{ startIndex: 13, type: 'white.pq' },
+			{ startIndex: 14, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '/* //*/ a',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' },
+			{ startIndex: 7, type: 'white.pq' },
+			{ startIndex: 8, type: 'identifier.pq' }
+		]
+	}],
+
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' },
+			{ startIndex: 1, type: 'white.pq' },
+			{ startIndex: 2, type: 'operators.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'number.pq' },
+			{ startIndex: 5, type: 'delimiter.pq' },
+			{ startIndex: 6, type: 'white.pq' },
+			{ startIndex: 7, type: 'comment.pq' }
+		]
+	}],
+
+	// Quoted Identifiers
+	[{
+		line: '#"Change Types"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.pq' }
+		]
+	}],
+
+	[{
+		line: '#"A  B" = 1+2,',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.pq' },
+			{ startIndex: 7, type: 'white.pq' },
+			{ startIndex: 8, type: 'operators.pq' },
+			{ startIndex: 9, type: 'white.pq' },
+			{ startIndex: 10, type: 'number.pq' },
+			{ startIndex: 11, type: 'operators.pq' },
+			{ startIndex: 12, type: 'number.pq' },
+			{ startIndex: 13, type: 'delimiter.pq' },
+		]
+	}],
+
+	[{
+		line: 'a = #"escap ed"+ 1',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 1, type: 'white.pq' },
+			{ startIndex: 2, type: 'operators.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'identifier.quote.pq' },
+			{ startIndex: 15, type: 'operators.pq' },
+			{ startIndex: 16, type: 'white.pq' },
+			{ startIndex: 17, type: 'number.pq' }
+		]
+	}],
+
+	// Number formats
+	[{
+		line: '0Xabc',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pq' }
+		]
+	}],
+
+	[{
+		line: '0xA',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pq' }
+		]
+	}],
+
+	[{
+		line: '1e1',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' }
+		]
+	}],
+
+	[{
+		line: '5 / 1.2e+2 + 0x1234abc',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' },
+			{ startIndex: 1, type: 'white.pq' },
+			{ startIndex: 2, type: 'operators.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'number.float.pq' },
+			{ startIndex: 10, type: 'white.pq' },
+			{ startIndex: 11, type: 'operators.pq' },
+			{ startIndex: 12, type: 'white.pq'},
+			{ startIndex: 13, type: 'number.hex.pq'}
+		]
+	}],
+
+	[{
+		line: '0xb *(.2)',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'operators.pq' },
+			{ startIndex: 5, type: 'delimiter.parenthesis.pq' },
+			{ startIndex: 6, type: 'number.float.pq' },
+			{ startIndex: 8, type: 'delimiter.parenthesis.pq' }
+		]
+	}],
+
+	[{
+		line: '1.23e34+1.2e-2-.3e2',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.pq' },
+			{ startIndex: 7, type: 'operators.pq' },
+			{ startIndex: 8, type: 'number.float.pq' },
+			{ startIndex: 14, type: 'operators.pq' },
+			{ startIndex: 15, type: 'number.float.pq' }
+		]
+	}],
+
+	// strings
+	[{
+		line: '  "string"',
+		tokens: [
+			{ startIndex: 0, type: 'white.pq' },
+			{ startIndex: 2, type: 'string.pq' }
+		]
+	}],
+
+	[{
+		line: '"string" & "another"',
+		tokens: [
+			{ startIndex: 0, type: 'string.pq' },
+			{ startIndex: 8, type: 'white.pq' },
+			{ startIndex: 9, type: 'operators.pq' },
+			{ startIndex: 10, type: 'white.pq' },
+			{ startIndex: 11, type: 'string.pq' }
+		]
+	}],
+
+	[{
+		line: '"with  ""escaped "" \'text',
+		tokens: [
+			{ startIndex: 0, type: 'string.pq' },
+			{ startIndex: 7, type: 'string.escape.pq' },
+			{ startIndex: 9, type: 'string.pq' },
+			{ startIndex: 17, type: 'string.escape.pq' },
+			{ startIndex: 19, type: 'string.pq' }
+		]
+	}],
+
+	// built-in keywords/identifiers
+	[{
+		line: 'And as Each each _',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'keyword.pq' },
+			{ startIndex: 6, type: 'white.pq' },
+			{ startIndex: 7, type: 'identifier.pq' },
+			{ startIndex: 11, type: 'white.pq' },
+			{ startIndex: 12, type: 'keyword.pq' },
+			{ startIndex: 16, type: 'white.pq' },
+			{ startIndex: 17, type: 'identifier.pq' }
+		]
+	}],
+
+	[{
+		line: '  #table({})',
+		tokens: [
+			{ startIndex: 0, type: 'white.pq' },
+			{ startIndex: 2, type: 'constructor.pq' },
+			{ startIndex: 8, type: 'delimiter.parenthesis.pq' },
+			{ startIndex: 9, type: 'delimiter.brackets.pq' },
+			{ startIndex: 11, type: "delimiter.parenthesis.pq" }
+		]
+	}],
+
+	[{
+		line: 'param as number',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 5, type: 'white.pq' },
+			{ startIndex: 6, type: 'keyword.pq' },
+			{ startIndex: 8, type: 'white.pq' },
+			{ startIndex: 9, type: 'type.pq' }
+		]
+	}],
+
+	[{
+		line: 'type table',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.pq' },
+			{ startIndex: 4, type: 'white.pq' },
+			{ startIndex: 5, type: 'type.pq' }
+		]
+	}],
+
+	[{
+		line: 'if (a = #nan) then null else a',
+		tokens: [
+			{ startIndex: 0, type: "keyword.pq" },
+			{ startIndex: 2, type: "white.pq" },
+			{ startIndex: 3, type: "delimiter.parenthesis.pq"},
+			{ startIndex: 4, type: "identifier.pq" },
+			{ startIndex: 5, type: "white.pq" },
+			{ startIndex: 6, type: "operators.pq" },
+			{ startIndex: 7, type: "white.pq" },
+			{ startIndex: 8, type: "constant.pq" },
+			{ startIndex: 12, type: "delimiter.parenthesis.pq"},
+			{ startIndex: 13, type: "white.pq" },
+			{ startIndex: 14, type: "keyword.pq" },
+			{ startIndex: 18, type: "white.pq" },
+			{ startIndex: 19, type: "type.pq" },
+			{ startIndex: 23, type: "white.pq" },
+			{ startIndex: 24, type: "keyword.pq" },
+			{ startIndex: 28, type: "white.pq" },
+			{ startIndex: 29, type: "identifier.pq" },
+		]
+	}],
+
+	// built-ins
+	[{
+		line: 'Text.From(1)',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.function.pq' },
+			{ startIndex: 9, type: 'delimiter.parenthesis.pq'},
+			{ startIndex: 10, type: 'number.pq' },
+			{ startIndex: 11, type: 'delimiter.parenthesis.pq'}
+		]
+	}],
+
+	[{
+		line: 'Text.ToBinary("123", BinaryEncoding.Base64)',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.function.pq' },
+			{ startIndex: 13, type: 'delimiter.parenthesis.pq' },
+			{ startIndex: 14, type: 'string.pq' },
+			{ startIndex: 19, type: 'delimiter.pq' },
+			{ startIndex: 20, type: 'white.pq' },
+			{ startIndex: 21, type: 'constant.pq' },
+			{ startIndex: 42, type: 'delimiter.parenthesis.pq' }
+		]
+	}],
+
+	[{
+		line: 'Int8.Type',
+		tokens: [
+			{ startIndex: 0, type: 'type.pq' }
+		]
+	}],
+
+	[{
+		line: 'DB2.Database',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.function.pq' }
+		]
+	}],
+
+	[{
+		line: 'RelativePosition.Type',
+		tokens: [
+			{ startIndex: 0, type: 'type.pq' }
+		]
+	}],
+
+	// other statements
+	[{
+		line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;',
+		tokens: [
+			{ startIndex: 0, type: "delimiter.square.pq" },
+			{ startIndex: 1, type: "identifier.pq" },
+			{ startIndex: 8, type: "operators.pq" },
+			{ startIndex: 9, type: "string.pq" },
+			{ startIndex: 18, type: "delimiter.square.pq" },
+			{ startIndex: 19, type: "white.pq" },
+			{ startIndex: 20, type: "keyword.pq" },
+			{ startIndex: 27, type: "white.pq" },
+			{ startIndex: 28, type: "identifier.pq" },
+			{ startIndex: 31, type: "delimiter.pq" },
+			{ startIndex: 32, type: "white.pq" },
+			{ startIndex: 33, type: "keyword.pq" },
+			{ startIndex: 39, type: "white.pq" },
+			{ startIndex: 40, type: "identifier.pq" },
+			{ startIndex: 51, type: "white.pq" },
+			{ startIndex: 52, type: "operators.pq" },
+			{ startIndex: 53, type: "white.pq" },
+			{ startIndex: 54, type: "number.pq" },
+			{ startIndex: 55, type: "delimiter.pq" }
+		]
+	}],
+
+	[{
+		line: 'isFunctionthen = 1;// comment',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 14, type: 'white.pq' },
+			{ startIndex: 15, type: 'operators.pq' },
+			{ startIndex: 16, type: 'white.pq' },
+			{ startIndex: 17, type: 'number.pq' },
+			{ startIndex: 18, type: 'delimiter.pq' },
+			{ startIndex: 19, type: 'comment.pq' },
+		]
+	}],
+
+	[{
+		line: '@RecursiveFunction()+@Rec.Func()',
+		tokens: [
+			{ startIndex: 0, type: 'operators.pq' },
+			{ startIndex: 1, type: 'identifier.pq' },
+			{ startIndex: 18, type: "delimiter.parenthesis.pq"},
+			{ startIndex: 20, type: 'operators.pq' },
+			{ startIndex: 22, type: 'identifier.pq' },
+			{ startIndex: 30, type: "delimiter.parenthesis.pq"},
+		]
+	}],
+]);

src/powerquery/powerquery.ts

@@ -0,0 +1,934 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['[', ']'], ['(', ')'], ['{', '}']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment', 'identifier'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment', 'identifier'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment', 'identifier'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment', 'identifier'] },
+	]
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.pq',
+	ignoreCase: false,
+
+	brackets: [
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '{', close: '}', token: 'delimiter.brackets' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	operatorKeywords: [
+		"and",
+		"not",
+		"or"
+	],
+
+	keywords: [
+		"as",
+		"each",
+		"else",
+		"error",
+		"false",
+		"if",
+		"in",
+		"is",
+		"let",
+		"meta",
+		"otherwise",
+		"section",
+		"shared",
+		"then",
+		"true",
+		"try",
+		"type"
+	],
+
+	constructors: [
+		"#binary",
+		"#date",
+		"#datetime",
+		"#datetimezone",
+		"#duration",
+		"#table",
+		"#time"
+	],
+
+	constants: [
+		"#infinity",
+		"#nan",
+		"#sections",
+		"#shared"
+	],
+
+	typeKeywords: [
+		"action",
+		"any",
+		"anynonnull",
+		"none",
+		"null",
+		"logical",
+		"number",
+		"time",
+		"date",
+		"datetime",
+		"datetimezone",
+		"duration",
+		"text",
+		"binary",
+		"list",
+		"record",
+		"table",
+		"function"
+	],
+
+	builtinFunctions: [
+		"Access.Database",
+		"Action.Return",
+		"Action.Sequence",
+		"Action.Try",
+		"ActiveDirectory.Domains",
+		"AdoDotNet.DataSource",
+		"AdoDotNet.Query",
+		"AdobeAnalytics.Cubes",
+		"AnalysisServices.Database",
+		"AnalysisServices.Databases",
+		"AzureStorage.BlobContents",
+		"AzureStorage.Blobs",
+		"AzureStorage.Tables",
+		"Binary.Buffer",
+		"Binary.Combine",
+		"Binary.Compress",
+		"Binary.Decompress",
+		"Binary.End",
+		"Binary.From",
+		"Binary.FromList",
+		"Binary.FromText",
+		"Binary.InferContentType",
+		"Binary.Length",
+		"Binary.ToList",
+		"Binary.ToText",
+		"BinaryFormat.7BitEncodedSignedInteger",
+		"BinaryFormat.7BitEncodedUnsignedInteger",
+		"BinaryFormat.Binary",
+		"BinaryFormat.Byte",
+		"BinaryFormat.ByteOrder",
+		"BinaryFormat.Choice",
+		"BinaryFormat.Decimal",
+		"BinaryFormat.Double",
+		"BinaryFormat.Group",
+		"BinaryFormat.Length",
+		"BinaryFormat.List",
+		"BinaryFormat.Null",
+		"BinaryFormat.Record",
+		"BinaryFormat.SignedInteger16",
+		"BinaryFormat.SignedInteger32",
+		"BinaryFormat.SignedInteger64",
+		"BinaryFormat.Single",
+		"BinaryFormat.Text",
+		"BinaryFormat.Transform",
+		"BinaryFormat.UnsignedInteger16",
+		"BinaryFormat.UnsignedInteger32",
+		"BinaryFormat.UnsignedInteger64",
+		"Byte.From",
+		"Character.FromNumber",
+		"Character.ToNumber",
+		"Combiner.CombineTextByDelimiter",
+		"Combiner.CombineTextByEachDelimiter",
+		"Combiner.CombineTextByLengths",
+		"Combiner.CombineTextByPositions",
+		"Combiner.CombineTextByRanges",
+		"Comparer.Equals",
+		"Comparer.FromCulture",
+		"Comparer.Ordinal",
+		"Comparer.OrdinalIgnoreCase",
+		"Csv.Document",
+		"Cube.AddAndExpandDimensionColumn",
+		"Cube.AddMeasureColumn",
+		"Cube.ApplyParameter",
+		"Cube.AttributeMemberId",
+		"Cube.AttributeMemberProperty",
+		"Cube.CollapseAndRemoveColumns",
+		"Cube.Dimensions",
+		"Cube.DisplayFolders",
+		"Cube.Measures",
+		"Cube.Parameters",
+		"Cube.Properties",
+		"Cube.PropertyKey",
+		"Cube.ReplaceDimensions",
+		"Cube.Transform",
+		"Currency.From",
+		"DB2.Database",
+		"Date.AddDays",
+		"Date.AddMonths",
+		"Date.AddQuarters",
+		"Date.AddWeeks",
+		"Date.AddYears",
+		"Date.Day",
+		"Date.DayOfWeek",
+		"Date.DayOfWeekName",
+		"Date.DayOfYear",
+		"Date.DaysInMonth",
+		"Date.EndOfDay",
+		"Date.EndOfMonth",
+		"Date.EndOfQuarter",
+		"Date.EndOfWeek",
+		"Date.EndOfYear",
+		"Date.From",
+		"Date.FromText",
+		"Date.IsInCurrentDay",
+		"Date.IsInCurrentMonth",
+		"Date.IsInCurrentQuarter",
+		"Date.IsInCurrentWeek",
+		"Date.IsInCurrentYear",
+		"Date.IsInNextDay",
+		"Date.IsInNextMonth",
+		"Date.IsInNextNDays",
+		"Date.IsInNextNMonths",
+		"Date.IsInNextNQuarters",
+		"Date.IsInNextNWeeks",
+		"Date.IsInNextNYears",
+		"Date.IsInNextQuarter",
+		"Date.IsInNextWeek",
+		"Date.IsInNextYear",
+		"Date.IsInPreviousDay",
+		"Date.IsInPreviousMonth",
+		"Date.IsInPreviousNDays",
+		"Date.IsInPreviousNMonths",
+		"Date.IsInPreviousNQuarters",
+		"Date.IsInPreviousNWeeks",
+		"Date.IsInPreviousNYears",
+		"Date.IsInPreviousQuarter",
+		"Date.IsInPreviousWeek",
+		"Date.IsInPreviousYear",
+		"Date.IsInYearToDate",
+		"Date.IsLeapYear",
+		"Date.Month",
+		"Date.MonthName",
+		"Date.QuarterOfYear",
+		"Date.StartOfDay",
+		"Date.StartOfMonth",
+		"Date.StartOfQuarter",
+		"Date.StartOfWeek",
+		"Date.StartOfYear",
+		"Date.ToRecord",
+		"Date.ToText",
+		"Date.WeekOfMonth",
+		"Date.WeekOfYear",
+		"Date.Year",
+		"DateTime.AddZone",
+		"DateTime.Date",
+		"DateTime.FixedLocalNow",
+		"DateTime.From",
+		"DateTime.FromFileTime",
+		"DateTime.FromText",
+		"DateTime.IsInCurrentHour",
+		"DateTime.IsInCurrentMinute",
+		"DateTime.IsInCurrentSecond",
+		"DateTime.IsInNextHour",
+		"DateTime.IsInNextMinute",
+		"DateTime.IsInNextNHours",
+		"DateTime.IsInNextNMinutes",
+		"DateTime.IsInNextNSeconds",
+		"DateTime.IsInNextSecond",
+		"DateTime.IsInPreviousHour",
+		"DateTime.IsInPreviousMinute",
+		"DateTime.IsInPreviousNHours",
+		"DateTime.IsInPreviousNMinutes",
+		"DateTime.IsInPreviousNSeconds",
+		"DateTime.IsInPreviousSecond",
+		"DateTime.LocalNow",
+		"DateTime.Time",
+		"DateTime.ToRecord",
+		"DateTime.ToText",
+		"DateTimeZone.FixedLocalNow",
+		"DateTimeZone.FixedUtcNow",
+		"DateTimeZone.From",
+		"DateTimeZone.FromFileTime",
+		"DateTimeZone.FromText",
+		"DateTimeZone.LocalNow",
+		"DateTimeZone.RemoveZone",
+		"DateTimeZone.SwitchZone",
+		"DateTimeZone.ToLocal",
+		"DateTimeZone.ToRecord",
+		"DateTimeZone.ToText",
+		"DateTimeZone.ToUtc",
+		"DateTimeZone.UtcNow",
+		"DateTimeZone.ZoneHours",
+		"DateTimeZone.ZoneMinutes",
+		"Decimal.From",
+		"Diagnostics.ActivityId",
+		"Diagnostics.Trace",
+		"DirectQueryCapabilities.From",
+		"Double.From",
+		"Duration.Days",
+		"Duration.From",
+		"Duration.FromText",
+		"Duration.Hours",
+		"Duration.Minutes",
+		"Duration.Seconds",
+		"Duration.ToRecord",
+		"Duration.ToText",
+		"Duration.TotalDays",
+		"Duration.TotalHours",
+		"Duration.TotalMinutes",
+		"Duration.TotalSeconds",
+		"Embedded.Value",
+		"Error.Record",
+		"Excel.CurrentWorkbook",
+		"Excel.Workbook",
+		"Exchange.Contents",
+		"Expression.Constant",
+		"Expression.Evaluate",
+		"Expression.Identifier",
+		"Facebook.Graph",
+		"File.Contents",
+		"Folder.Contents",
+		"Folder.Files",
+		"Function.From",
+		"Function.Invoke",
+		"Function.InvokeAfter",
+		"Function.IsDataSource",
+		"GoogleAnalytics.Accounts",
+		"Guid.From",
+		"HdInsight.Containers",
+		"HdInsight.Contents",
+		"HdInsight.Files",
+		"Hdfs.Contents",
+		"Hdfs.Files",
+		"Informix.Database",
+		"Int16.From",
+		"Int32.From",
+		"Int64.From",
+		"Int8.From",
+		"ItemExpression.From",
+		"Json.Document",
+		"Json.FromValue",
+		"Lines.FromBinary",
+		"Lines.FromText",
+		"Lines.ToBinary",
+		"Lines.ToText",
+		"List.Accumulate",
+		"List.AllTrue",
+		"List.Alternate",
+		"List.AnyTrue",
+		"List.Average",
+		"List.Buffer",
+		"List.Combine",
+		"List.Contains",
+		"List.ContainsAll",
+		"List.ContainsAny",
+		"List.Count",
+		"List.Covariance",
+		"List.DateTimeZones",
+		"List.DateTimes",
+		"List.Dates",
+		"List.Difference",
+		"List.Distinct",
+		"List.Durations",
+		"List.FindText",
+		"List.First",
+		"List.FirstN",
+		"List.Generate",
+		"List.InsertRange",
+		"List.Intersect",
+		"List.IsDistinct",
+		"List.IsEmpty",
+		"List.Last",
+		"List.LastN",
+		"List.MatchesAll",
+		"List.MatchesAny",
+		"List.Max",
+		"List.MaxN",
+		"List.Median",
+		"List.Min",
+		"List.MinN",
+		"List.Mode",
+		"List.Modes",
+		"List.NonNullCount",
+		"List.Numbers",
+		"List.PositionOf",
+		"List.PositionOfAny",
+		"List.Positions",
+		"List.Product",
+		"List.Random",
+		"List.Range",
+		"List.RemoveFirstN",
+		"List.RemoveItems",
+		"List.RemoveLastN",
+		"List.RemoveMatchingItems",
+		"List.RemoveNulls",
+		"List.RemoveRange",
+		"List.Repeat",
+		"List.ReplaceMatchingItems",
+		"List.ReplaceRange",
+		"List.ReplaceValue",
+		"List.Reverse",
+		"List.Select",
+		"List.Single",
+		"List.SingleOrDefault",
+		"List.Skip",
+		"List.Sort",
+		"List.StandardDeviation",
+		"List.Sum",
+		"List.Times",
+		"List.Transform",
+		"List.TransformMany",
+		"List.Union",
+		"List.Zip",
+		"Logical.From",
+		"Logical.FromText",
+		"Logical.ToText",
+		"MQ.Queue",
+		"MySQL.Database",
+		"Number.Abs",
+		"Number.Acos",
+		"Number.Asin",
+		"Number.Atan",
+		"Number.Atan2",
+		"Number.BitwiseAnd",
+		"Number.BitwiseNot",
+		"Number.BitwiseOr",
+		"Number.BitwiseShiftLeft",
+		"Number.BitwiseShiftRight",
+		"Number.BitwiseXor",
+		"Number.Combinations",
+		"Number.Cos",
+		"Number.Cosh",
+		"Number.Exp",
+		"Number.Factorial",
+		"Number.From",
+		"Number.FromText",
+		"Number.IntegerDivide",
+		"Number.IsEven",
+		"Number.IsNaN",
+		"Number.IsOdd",
+		"Number.Ln",
+		"Number.Log",
+		"Number.Log10",
+		"Number.Mod",
+		"Number.Permutations",
+		"Number.Power",
+		"Number.Random",
+		"Number.RandomBetween",
+		"Number.Round",
+		"Number.RoundAwayFromZero",
+		"Number.RoundDown",
+		"Number.RoundTowardZero",
+		"Number.RoundUp",
+		"Number.Sign",
+		"Number.Sin",
+		"Number.Sinh",
+		"Number.Sqrt",
+		"Number.Tan",
+		"Number.Tanh",
+		"Number.ToText",
+		"OData.Feed",
+		"Odbc.DataSource",
+		"Odbc.Query",
+		"OleDb.DataSource",
+		"OleDb.Query",
+		"Oracle.Database",
+		"Percentage.From",
+		"PostgreSQL.Database",
+		"RData.FromBinary",
+		"Record.AddField",
+		"Record.Combine",
+		"Record.Field",
+		"Record.FieldCount",
+		"Record.FieldNames",
+		"Record.FieldOrDefault",
+		"Record.FieldValues",
+		"Record.FromList",
+		"Record.FromTable",
+		"Record.HasFields",
+		"Record.RemoveFields",
+		"Record.RenameFields",
+		"Record.ReorderFields",
+		"Record.SelectFields",
+		"Record.ToList",
+		"Record.ToTable",
+		"Record.TransformFields",
+		"Replacer.ReplaceText",
+		"Replacer.ReplaceValue",
+		"RowExpression.Column",
+		"RowExpression.From",
+		"Salesforce.Data",
+		"Salesforce.Reports",
+		"SapBusinessWarehouse.Cubes",
+		"SapHana.Database",
+		"SharePoint.Contents",
+		"SharePoint.Files",
+		"SharePoint.Tables",
+		"Single.From",
+		"Soda.Feed",
+		"Splitter.SplitByNothing",
+		"Splitter.SplitTextByAnyDelimiter",
+		"Splitter.SplitTextByDelimiter",
+		"Splitter.SplitTextByEachDelimiter",
+		"Splitter.SplitTextByLengths",
+		"Splitter.SplitTextByPositions",
+		"Splitter.SplitTextByRanges",
+		"Splitter.SplitTextByRepeatedLengths",
+		"Splitter.SplitTextByWhitespace",
+		"Sql.Database",
+		"Sql.Databases",
+		"SqlExpression.SchemaFrom",
+		"SqlExpression.ToExpression",
+		"Sybase.Database",
+		"Table.AddColumn",
+		"Table.AddIndexColumn",
+		"Table.AddJoinColumn",
+		"Table.AddKey",
+		"Table.AggregateTableColumn",
+		"Table.AlternateRows",
+		"Table.Buffer",
+		"Table.Column",
+		"Table.ColumnCount",
+		"Table.ColumnNames",
+		"Table.ColumnsOfType",
+		"Table.Combine",
+		"Table.CombineColumns",
+		"Table.Contains",
+		"Table.ContainsAll",
+		"Table.ContainsAny",
+		"Table.DemoteHeaders",
+		"Table.Distinct",
+		"Table.DuplicateColumn",
+		"Table.ExpandListColumn",
+		"Table.ExpandRecordColumn",
+		"Table.ExpandTableColumn",
+		"Table.FillDown",
+		"Table.FillUp",
+		"Table.FilterWithDataTable",
+		"Table.FindText",
+		"Table.First",
+		"Table.FirstN",
+		"Table.FirstValue",
+		"Table.FromColumns",
+		"Table.FromList",
+		"Table.FromPartitions",
+		"Table.FromRecords",
+		"Table.FromRows",
+		"Table.FromValue",
+		"Table.Group",
+		"Table.HasColumns",
+		"Table.InsertRows",
+		"Table.IsDistinct",
+		"Table.IsEmpty",
+		"Table.Join",
+		"Table.Keys",
+		"Table.Last",
+		"Table.LastN",
+		"Table.MatchesAllRows",
+		"Table.MatchesAnyRows",
+		"Table.Max",
+		"Table.MaxN",
+		"Table.Min",
+		"Table.MinN",
+		"Table.NestedJoin",
+		"Table.Partition",
+		"Table.PartitionValues",
+		"Table.Pivot",
+		"Table.PositionOf",
+		"Table.PositionOfAny",
+		"Table.PrefixColumns",
+		"Table.Profile",
+		"Table.PromoteHeaders",
+		"Table.Range",
+		"Table.RemoveColumns",
+		"Table.RemoveFirstN",
+		"Table.RemoveLastN",
+		"Table.RemoveMatchingRows",
+		"Table.RemoveRows",
+		"Table.RemoveRowsWithErrors",
+		"Table.RenameColumns",
+		"Table.ReorderColumns",
+		"Table.Repeat",
+		"Table.ReplaceErrorValues",
+		"Table.ReplaceKeys",
+		"Table.ReplaceMatchingRows",
+		"Table.ReplaceRelationshipIdentity",
+		"Table.ReplaceRows",
+		"Table.ReplaceValue",
+		"Table.ReverseRows",
+		"Table.RowCount",
+		"Table.Schema",
+		"Table.SelectColumns",
+		"Table.SelectRows",
+		"Table.SelectRowsWithErrors",
+		"Table.SingleRow",
+		"Table.Skip",
+		"Table.Sort",
+		"Table.SplitColumn",
+		"Table.ToColumns",
+		"Table.ToList",
+		"Table.ToRecords",
+		"Table.ToRows",
+		"Table.TransformColumnNames",
+		"Table.TransformColumnTypes",
+		"Table.TransformColumns",
+		"Table.TransformRows",
+		"Table.Transpose",
+		"Table.Unpivot",
+		"Table.UnpivotOtherColumns",
+		"Table.View",
+		"Table.ViewFunction",
+		"TableAction.DeleteRows",
+		"TableAction.InsertRows",
+		"TableAction.UpdateRows",
+		"Tables.GetRelationships",
+		"Teradata.Database",
+		"Text.AfterDelimiter",
+		"Text.At",
+		"Text.BeforeDelimiter",
+		"Text.BetweenDelimiters",
+		"Text.Clean",
+		"Text.Combine",
+		"Text.Contains",
+		"Text.End",
+		"Text.EndsWith",
+		"Text.Format",
+		"Text.From",
+		"Text.FromBinary",
+		"Text.Insert",
+		"Text.Length",
+		"Text.Lower",
+		"Text.Middle",
+		"Text.NewGuid",
+		"Text.PadEnd",
+		"Text.PadStart",
+		"Text.PositionOf",
+		"Text.PositionOfAny",
+		"Text.Proper",
+		"Text.Range",
+		"Text.Remove",
+		"Text.RemoveRange",
+		"Text.Repeat",
+		"Text.Replace",
+		"Text.ReplaceRange",
+		"Text.Select",
+		"Text.Split",
+		"Text.SplitAny",
+		"Text.Start",
+		"Text.StartsWith",
+		"Text.ToBinary",
+		"Text.ToList",
+		"Text.Trim",
+		"Text.TrimEnd",
+		"Text.TrimStart",
+		"Text.Upper",
+		"Time.EndOfHour",
+		"Time.From",
+		"Time.FromText",
+		"Time.Hour",
+		"Time.Minute",
+		"Time.Second",
+		"Time.StartOfHour",
+		"Time.ToRecord",
+		"Time.ToText",
+		"Type.AddTableKey",
+		"Type.ClosedRecord",
+		"Type.Facets",
+		"Type.ForFunction",
+		"Type.ForRecord",
+		"Type.FunctionParameters",
+		"Type.FunctionRequiredParameters",
+		"Type.FunctionReturn",
+		"Type.Is",
+		"Type.IsNullable",
+		"Type.IsOpenRecord",
+		"Type.ListItem",
+		"Type.NonNullable",
+		"Type.OpenRecord",
+		"Type.RecordFields",
+		"Type.ReplaceFacets",
+		"Type.ReplaceTableKeys",
+		"Type.TableColumn",
+		"Type.TableKeys",
+		"Type.TableRow",
+		"Type.TableSchema",
+		"Type.Union",
+		"Uri.BuildQueryString",
+		"Uri.Combine",
+		"Uri.EscapeDataString",
+		"Uri.Parts",
+		"Value.Add",
+		"Value.As",
+		"Value.Compare",
+		"Value.Divide",
+		"Value.Equals",
+		"Value.Firewall",
+		"Value.FromText",
+		"Value.Is",
+		"Value.Metadata",
+		"Value.Multiply",
+		"Value.NativeQuery",
+		"Value.NullableEquals",
+		"Value.RemoveMetadata",
+		"Value.ReplaceMetadata",
+		"Value.ReplaceType",
+		"Value.Subtract",
+		"Value.Type",
+		"ValueAction.NativeStatement",
+		"ValueAction.Replace",
+		"Variable.Value",
+		"Web.Contents",
+		"Web.Page",
+		"WebAction.Request",
+		"Xml.Document",
+		"Xml.Tables"
+	],
+
+	builtinConstants: [
+		"BinaryEncoding.Base64",
+		"BinaryEncoding.Hex",
+		"BinaryOccurrence.Optional",
+		"BinaryOccurrence.Repeating",
+		"BinaryOccurrence.Required",
+		"ByteOrder.BigEndian",
+		"ByteOrder.LittleEndian",
+		"Compression.Deflate",
+		"Compression.GZip",
+		"CsvStyle.QuoteAfterDelimiter",
+		"CsvStyle.QuoteAlways",
+		"Culture.Current",
+		"Day.Friday",
+		"Day.Monday",
+		"Day.Saturday",
+		"Day.Sunday",
+		"Day.Thursday",
+		"Day.Tuesday",
+		"Day.Wednesday",
+		"ExtraValues.Error",
+		"ExtraValues.Ignore",
+		"ExtraValues.List",
+		"GroupKind.Global",
+		"GroupKind.Local",
+		"JoinAlgorithm.Dynamic",
+		"JoinAlgorithm.LeftHash",
+		"JoinAlgorithm.LeftIndex",
+		"JoinAlgorithm.PairwiseHash",
+		"JoinAlgorithm.RightHash",
+		"JoinAlgorithm.RightIndex",
+		"JoinAlgorithm.SortMerge",
+		"JoinKind.FullOuter",
+		"JoinKind.Inner",
+		"JoinKind.LeftAnti",
+		"JoinKind.LeftOuter",
+		"JoinKind.RightAnti",
+		"JoinKind.RightOuter",
+		"JoinSide.Left",
+		"JoinSide.Right",
+		"MissingField.Error",
+		"MissingField.Ignore",
+		"MissingField.UseNull",
+		"Number.E",
+		"Number.Epsilon",
+		"Number.NaN",
+		"Number.NegativeInfinity",
+		"Number.PI",
+		"Number.PositiveInfinity",
+		"Occurrence.All",
+		"Occurrence.First",
+		"Occurrence.Last",
+		"Occurrence.Optional",
+		"Occurrence.Repeating",
+		"Occurrence.Required",
+		"Order.Ascending",
+		"Order.Descending",
+		"Precision.Decimal",
+		"Precision.Double",
+		"QuoteStyle.Csv",
+		"QuoteStyle.None",
+		"RelativePosition.FromEnd",
+		"RelativePosition.FromStart",
+		"RoundingMode.AwayFromZero",
+		"RoundingMode.Down",
+		"RoundingMode.ToEven",
+		"RoundingMode.TowardZero",
+		"RoundingMode.Up",
+		"SapHanaDistribution.All",
+		"SapHanaDistribution.Connection",
+		"SapHanaDistribution.Off",
+		"SapHanaDistribution.Statement",
+		"SapHanaRangeOperator.Equals",
+		"SapHanaRangeOperator.GreaterThan",
+		"SapHanaRangeOperator.GreaterThanOrEquals",
+		"SapHanaRangeOperator.LessThan",
+		"SapHanaRangeOperator.LessThanOrEquals",
+		"SapHanaRangeOperator.NotEquals",
+		"TextEncoding.Ascii",
+		"TextEncoding.BigEndianUnicode",
+		"TextEncoding.Unicode",
+		"TextEncoding.Utf16",
+		"TextEncoding.Utf8",
+		"TextEncoding.Windows",
+		"TraceLevel.Critical",
+		"TraceLevel.Error",
+		"TraceLevel.Information",
+		"TraceLevel.Verbose",
+		"TraceLevel.Warning",
+		"WebMethod.Delete",
+		"WebMethod.Get",
+		"WebMethod.Head",
+		"WebMethod.Patch",
+		"WebMethod.Post",
+		"WebMethod.Put"
+	],
+
+	builtinTypes: [
+		"Action.Type",
+		"Any.Type",
+		"Binary.Type",
+		"BinaryEncoding.Type",
+		"BinaryOccurrence.Type",
+		"Byte.Type",
+		"ByteOrder.Type",
+		"Character.Type",
+		"Compression.Type",
+		"CsvStyle.Type",
+		"Currency.Type",
+		"Date.Type",
+		"DateTime.Type",
+		"DateTimeZone.Type",
+		"Day.Type",
+		"Decimal.Type",
+		"Double.Type",
+		"Duration.Type",
+		"ExtraValues.Type",
+		"Function.Type",
+		"GroupKind.Type",
+		"Guid.Type",
+		"Int16.Type",
+		"Int32.Type",
+		"Int64.Type",
+		"Int8.Type",
+		"JoinAlgorithm.Type",
+		"JoinKind.Type",
+		"JoinSide.Type",
+		"List.Type",
+		"Logical.Type",
+		"MissingField.Type",
+		"None.Type",
+		"Null.Type",
+		"Number.Type",
+		"Occurrence.Type",
+		"Order.Type",
+		"Password.Type",
+		"Percentage.Type",
+		"Precision.Type",
+		"QuoteStyle.Type",
+		"Record.Type",
+		"RelativePosition.Type",
+		"RoundingMode.Type",
+		"SapHanaDistribution.Type",
+		"SapHanaRangeOperator.Type",
+		"Single.Type",
+		"Table.Type",
+		"Text.Type",
+		"TextEncoding.Type",
+		"Time.Type",
+		"TraceLevel.Type",
+		"Type.Type",
+		"Uri.Type",
+		"WebMethod.Type"
+	],
+
+	tokenizer: {
+		root: [
+			// quoted identifier
+			[/#"[\w \.]+"/, "identifier.quote"],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, "number.float"],
+			[/0[xX][0-9a-fA-F]+/, "number.hex"],
+			[/\d+([eE][\-+]?\d+)?/, "number"],
+
+			// keywords
+			[/(#?[a-z]+)\b/,
+				{
+					cases: {
+						"@typeKeywords": "type",
+						"@keywords": "keyword",
+						"@constants": "constant",
+						"@constructors": "constructor",
+						"@operatorKeywords": "operators",
+						"@default": "identifier"
+					}
+				}
+			],
+
+			// built-in types
+			[/\b([A-Z][a-zA-Z0-9]+\.Type)\b/,
+				{
+					cases: {
+						"@builtinTypes": "type",
+						"@default": "identifier"
+					}
+				}
+			],
+
+			// other built-ins
+			[/\b([A-Z][a-zA-Z0-9]+\.[A-Z][a-zA-Z0-9]+)\b/,
+				{
+					cases: {
+						"@builtinFunctions": "keyword.function",
+						"@builtinConstants": "constant",
+						"@default": "identifier"
+					}
+				}
+			],
+
+			// other identifiers
+			[/\b([a-zA-Z_][\w\.]*)\b/, "identifier"],
+
+			{ include: "@whitespace" },
+			{ include: "@comments" },
+			{ include: "@strings" },
+
+			[/[{}()\[\]]/, "@brackets"],
+			[/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operators"],
+			[/[,;]/, "delimiter"],
+		],
+
+		whitespace: [
+			[/\s+/, "white"]
+		],
+
+		comments: [
+			["\\/\\*", "comment", "@comment"],
+			["\\/\\/+.*", "comment"]
+		],
+
+		comment: [
+			["\\*\\/", "comment", "@pop"],
+			[".", "comment"]
+		],
+
+		strings: [
+			["\"", "string", "@string"]
+		],
+
+		string: [
+			["\"\"", "string.escape"],
+			["\"", "string", "@pop"],
+			[".", "string"]
+		]
+	}
+};

src/scheme/scheme.test.ts

@@ -8,84 +8,84 @@
 import { testTokenization } from '../test/testRunner';
 
 testTokenization('scheme', [
-    // Keywords
-    [
-        {
-            line: 'define-macro some',
-            tokens: [
-                { startIndex: 0, type: 'keyword.scheme' },
-                { startIndex: 12, type: 'white.scheme' },
-                { startIndex: 13, type: 'variable.scheme' },
-            ],
-        },
-    ],
+	// Keywords
+	[
+		{
+			line: 'define-macro some',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.scheme' },
+				{ startIndex: 12, type: 'white.scheme' },
+				{ startIndex: 13, type: 'variable.scheme' },
+			],
+		},
+	],
 
-    // comments
-    [
-        {
-            line: '; comment',
-            tokens: [{ startIndex: 0, type: 'comment.scheme' }],
-        },
-    ],
-    [
-        {
-            line: '|# comment',
-            tokens: [{ startIndex: 0, type: 'comment.scheme' }],
-        },
-        {
-            line: 'multiline',
-            tokens: [{ startIndex: 0, type: 'comment.scheme' }],
-        },
-        {
-            line: '#| cons',
-            tokens: [
-                { startIndex: 0, type: 'comment.scheme' },
-                { startIndex: 2, type: 'white.scheme' },
-                { startIndex: 3, type: 'keyword.scheme' },
-            ],
-        },
-    ],
+	// comments
+	[
+		{
+			line: '; comment',
+			tokens: [{ startIndex: 0, type: 'comment.scheme' }],
+		},
+	],
+	[
+		{
+			line: '#| comment',
+			tokens: [{ startIndex: 0, type: 'comment.scheme' }],
+		},
+		{
+			line: 'multiline',
+			tokens: [{ startIndex: 0, type: 'comment.scheme' }],
+		},
+		{
+			line: '|# cons',
+			tokens: [
+				{ startIndex: 0, type: 'comment.scheme' },
+				{ startIndex: 2, type: 'white.scheme' },
+				{ startIndex: 3, type: 'keyword.scheme' },
+			],
+		},
+	],
 
-    // strings
-    [
-        {
-            line: '"\\n string "',
-            tokens: [
-                { startIndex: 0, type: 'string.scheme' },
-                { startIndex: 1, type: 'string.escape.scheme' },
-                { startIndex: 3, type: 'string.scheme' },
-            ],
-        },
-    ],
-    [
-        {
-            line: '" string \\',
-            tokens: [{ startIndex: 0, type: 'string.scheme' }],
-        },
-        {
-            line: 'multiline',
-            tokens: [{ startIndex: 0, type: 'string.scheme' }],
-        },
-        {
-            line: ' ',
-            tokens: [
-                // previous line needs to be terminated with \
-                { startIndex: 0, type: 'white.scheme' },
-            ],
-        },
-    ],
+	// strings
+	[
+		{
+			line: '"\\n string "',
+			tokens: [
+				{ startIndex: 0, type: 'string.scheme' },
+				{ startIndex: 1, type: 'string.escape.scheme' },
+				{ startIndex: 3, type: 'string.scheme' },
+			],
+		},
+	],
+	[
+		{
+			line: '" string \\',
+			tokens: [{ startIndex: 0, type: 'string.scheme' }],
+		},
+		{
+			line: 'multiline',
+			tokens: [{ startIndex: 0, type: 'string.scheme' }],
+		},
+		{
+			line: ' ',
+			tokens: [
+				// previous line needs to be terminated with \
+				{ startIndex: 0, type: 'white.scheme' },
+			],
+		},
+	],
 
-    // numbers
-    [
-        {
-            line: '1e2',
-            tokens: [{ startIndex: 0, type: 'number.float.scheme' }],
-        },
-    ],
-    [
-        {
-            line: '#x03BB',
-            tokens: [{ startIndex: 0, type: 'number.hex.scheme' }],
-        },
-    ],
+	// numbers
+	[
+		{
+			line: '1e2',
+			tokens: [{ startIndex: 0, type: 'number.float.scheme' }],
+		},
+	],
+	[
+		{
+			line: '#x03BB',
+			tokens: [{ startIndex: 0, type: 'number.hex.scheme' }],
+		},
+	],
 ]);

src/scheme/scheme.ts

@@ -9,116 +9,119 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
 export const conf: IRichLanguageConfiguration = {
-    comments: {
-        lineComment: ';',
-        blockComment: ['#|', '|#'],
-    },
-
-    brackets: [['(', ')'], ['{', '}'], ['[', ']']],
-
-    autoClosingPairs: [
-        { open: '{', close: '}' },
-        { open: '[', close: ']' },
-        { open: '(', close: ')' },
-        { open: '"', close: '"' },
-    ],
-
-    surroundingPairs: [
-        { open: '{', close: '}' },
-        { open: '[', close: ']' },
-        { open: '(', close: ')' },
-        { open: '"', close: '"' },
-    ],
+	comments: {
+		lineComment: ';',
+		blockComment: ['#|', '|#'],
+	},
+
+	brackets: [['(', ')'], ['{', '}'], ['[', ']']],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+	],
+
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+	],
 };
 
 export const language = <ILanguage>{
-    defaultToken: '',
-    ignoreCase: true,
-    tokenPostfix: '.scheme',
-
-    brackets: [
-        { open: '(', close: ')', token: 'delimiter.parenthesis' },
-        { open: '{', close: '}', token: 'delimiter.curly' },
-        { open: '[', close: ']', token: 'delimiter.square' },
-    ],
-
-    keywords: [
-        'case',
-        'do',
-        'let',
-        'loop',
-        'if',
-        'else',
-        'when',
-        'cons',
-        'car',
-        'cdr',
-        'cond',
-        'lambda',
-        'lambda*',
-        'syntax-rules',
-        'format',
-        'set!',
-        'quote',
-        'eval',
-        'append',
-        'list',
-        'list?',
-        'member?',
-        'load',
-    ],
-
-    constants: ['#t', '#f'],
-
-    operators: ['eq?', 'eqv?', 'equal?', 'and', 'or', 'not', 'null?'],
-
-    tokenizer: {
-        root: [
-            [/#[xXoObB][0-9a-fA-F]+/, 'number.hex'],
-            [/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'],
-
-            [/(?:\b(?:(define|define-syntax|define-macro))\b)(\s+)((?:\w|\-|\!|\?)*)/, ['keyword', 'white', 'variable']],
-
-            [
-                /[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/,
-                {
-                    cases: {
-                        '@keywords': 'keyword',
-                        '@constants': 'constant',
-                        '@operators': 'operators',
-                        '@default': 'identifier',
-                    },
-                },
-            ],
-
-            { include: '@whitespace' },
-            { include: '@strings' },
-        ],
-
-        comment: [
-            [/[^\|#]+/, 'comment'],
-            [/\|\#/, 'comment', '@push'],
-            [/#\|/, 'comment', '@pop'],
-            [/[\|#]/, 'comment'],
-        ],
-
-        whitespace: [
-            [/[ \t\r\n]+/, 'white'],
-            [/\|\#/, 'comment', '@comment'],
-            [/;.*$/, 'comment'],
-        ],
-
-        strings: [
-            [/"$/, 'string', '@popall'],
-            [/"(?=.)/, 'string', '@multiLineString'],
-        ],
-
-        multiLineString: [
-            [/\\./, 'string.escape'],
-            [/"/, 'string', '@popall'],
-            [/.(?=.*")/, 'string'],
-            [/.*\\$/, 'string'],
-            [/.*$/, 'string', '@popall'],
-        ],
-    },
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.scheme',
+
+	brackets: [
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+	],
+
+	keywords: [
+		'case',
+		'do',
+		'let',
+		'loop',
+		'if',
+		'else',
+		'when',
+		'cons',
+		'car',
+		'cdr',
+		'cond',
+		'lambda',
+		'lambda*',
+		'syntax-rules',
+		'format',
+		'set!',
+		'quote',
+		'eval',
+		'append',
+		'list',
+		'list?',
+		'member?',
+		'load',
+	],
+
+	constants: ['#t', '#f'],
+
+	operators: ['eq?', 'eqv?', 'equal?', 'and', 'or', 'not', 'null?'],
+
+	tokenizer: {
+		root: [
+			[/#[xXoObB][0-9a-fA-F]+/, 'number.hex'],
+			[/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'],
+
+			[
+				/(?:\b(?:(define|define-syntax|define-macro))\b)(\s+)((?:\w|\-|\!|\?)*)/,
+				['keyword', 'white', 'variable'],
+			],
+
+			{ include: '@whitespace' },
+			{ include: '@strings' },
+
+			[
+				/[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@constants': 'constant',
+						'@operators': 'operators',
+						'@default': 'identifier',
+					},
+				},
+			],
+		],
+
+		comment: [
+			[/[^\|#]+/, 'comment'],
+			[/#\|/, 'comment', '@push'],
+			[/\|#/, 'comment', '@pop'],
+			[/[\|#]/, 'comment'],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+			[/#\|/, 'comment', '@comment'],
+			[/;.*$/, 'comment'],
+		],
+
+		strings: [
+			[/"$/, 'string', '@popall'],
+			[/"(?=.)/, 'string', '@multiLineString'],
+		],
+
+		multiLineString: [
+			[/\\./, 'string.escape'],
+			[/"/, 'string', '@popall'],
+			[/.(?=.*")/, 'string'],
+			[/.*\\$/, 'string'],
+			[/.*$/, 'string', '@popall'],
+		],
+	},
 };

src/typescript/typescript.contribution.ts

@@ -0,0 +1,18 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+registerLanguage({
+	id: 'typescript',
+	extensions: ['.ts', '.tsx'],
+	aliases: ['TypeScript', 'ts', 'typescript'],
+	mimetypes: ['text/typescript'],
+	loader: () => _monaco.Promise.wrap(<Promise<any>>import('./typescript'))
+});

src/typescript/typescript.test.ts

@@ -0,0 +1,557 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('typescript', [
+	// Keywords
+	[{
+		line: 'var x = function() { };',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'keyword.ts' },
+			{ startIndex: 16, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 18, type: '' },
+			{ startIndex: 19, type: 'delimiter.bracket.ts' },
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'delimiter.bracket.ts' },
+			{ startIndex: 22, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '    var    ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'keyword.ts' },
+			{ startIndex: 7, type: '' }
+		]
+	}],
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '// a comment /*',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '// a comment /**',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: 'var x = 1; // my comment // is a nice one',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.ts' },
+			{ startIndex: 9, type: 'delimiter.ts' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'comment.ts' }
+		]
+	}],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.ts' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.ts' },
+			{ startIndex: 32, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.ts' },
+			{ startIndex: 8, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'x = /*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.ts' }
+		]
+	}],
+
+	// Comments - range comment, multi lines
+	[{
+		line: '/* a multiline comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}, {
+		line: 'can actually span',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}, {
+		line: 'multiple lines */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* start a comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.ts' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}, {
+		line: 'and end it */ var a = 2;',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.ts' },
+			{ startIndex: 17, type: '' },
+			{ startIndex: 18, type: 'identifier.ts' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'delimiter.ts' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.ts' },
+			{ startIndex: 23, type: 'delimiter.ts' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: 'var a = \'a\';',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.ts' },
+			{ startIndex: 11, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '"use strict";',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 12, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'b = a + " \'cool\'  "',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.ts' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 10, type: 'string.escape.ts' },
+			{ startIndex: 12, type: 'string.ts' },
+			{ startIndex: 18, type: 'string.escape.ts' },
+			{ startIndex: 20, type: 'string.ts' },
+		]
+	}],
+
+	[{
+		line: '\'\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 2, type: 'string.invalid.ts' },
+		]
+	}],
+
+	[{
+		line: '\'\\\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 1, type: 'string.escape.ts' },
+			{ startIndex: 3, type: 'string.ts' },
+		]
+	}],
+
+	[{
+		line: '\'be careful \\not to escape\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 12, type: 'string.escape.ts' },
+			{ startIndex: 14, type: 'string.ts' },
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: ' 0',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: ' 0 ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.ts' },
+			{ startIndex: 2, type: '' }
+		]
+	}],
+
+	[{
+		line: '0 ',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: 'delimiter.ts' },
+			{ startIndex: 2, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 3, type: 'delimiter.ts' },
+			{ startIndex: 4, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '0123',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.ts' }
+		]
+	}],
+
+	[{
+		line: '01239',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.ts' },
+			{ startIndex: 4, type: 'number.ts' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: 'identifier.ts' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.ts' }
+		]
+	}],
+
+	// Regular Expressions
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '/**/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '/***/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.ts' }
+		]
+	}],
+
+	[{
+		line: '5 / 3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' }
+		]
+	}],
+
+	// Advanced regular expressions
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.ts' }
+		]
+	}],
+
+	[{
+		line: '1 / 2 / x / b;',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.ts' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'delimiter.ts' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.ts' },
+			{ startIndex: 13, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: 'a /ads/ b;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: 'identifier.ts' },
+			{ startIndex: 6, type: 'delimiter.ts' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.ts' },
+			{ startIndex: 9, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '1/(2/3)/2/3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.ts' },
+			{ startIndex: 1, type: 'delimiter.ts' },
+			{ startIndex: 2, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 3, type: 'number.ts' },
+			{ startIndex: 4, type: 'delimiter.ts' },
+			{ startIndex: 5, type: 'number.ts' },
+			{ startIndex: 6, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 7, type: 'delimiter.ts' },
+			{ startIndex: 8, type: 'number.ts' },
+			{ startIndex: 9, type: 'delimiter.ts' },
+			{ startIndex: 10, type: 'number.ts' },
+			{ startIndex: 11, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '{ key: 123 }',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'identifier.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.ts' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.bracket.ts' }
+		]
+	}],
+
+	[{
+		line: '[1,2,3]',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.square.ts' },
+			{ startIndex: 1, type: 'number.ts' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: 'number.ts' },
+			{ startIndex: 4, type: 'delimiter.ts' },
+			{ startIndex: 5, type: 'number.ts' },
+			{ startIndex: 6, type: 'delimiter.square.ts' }
+		]
+	}],
+
+	[{
+		line: 'foo(123);',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 3, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 4, type: 'number.ts' },
+			{ startIndex: 7, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 8, type: 'delimiter.ts' }
+		]
+	}],
+
+	[{
+		line: '{a:{b:[]}}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.ts' },
+			{ startIndex: 1, type: 'identifier.ts' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: 'delimiter.bracket.ts' },
+			{ startIndex: 4, type: 'identifier.ts' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: 'delimiter.square.ts' },
+			{ startIndex: 8, type: 'delimiter.bracket.ts' }
+		]
+	}],
+
+	[{
+		line: 'x = "[{()}]"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.ts' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.ts' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'string.ts' }
+		]
+	}],
+
+
+	[{
+		line: '`${5 + \'x\' + (<any>)3}a${4}`',
+		tokens: [
+			{ startIndex: 0, type: 'string.ts' },
+			{ startIndex: 1, type: 'delimiter.bracket.ts' },
+			{ startIndex: 3, type: 'number.ts' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter.ts' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'string.ts' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.ts' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 14, type: 'delimiter.angle.ts' },
+			{ startIndex: 15, type: 'keyword.ts' },
+			{ startIndex: 18, type: 'delimiter.angle.ts' },
+			{ startIndex: 19, type: 'delimiter.parenthesis.ts' },
+			{ startIndex: 20, type: 'number.ts' },
+			{ startIndex: 21, type: 'delimiter.bracket.ts' },
+			{ startIndex: 22, type: 'string.ts' },
+			{ startIndex: 23, type: 'delimiter.bracket.ts' },
+			{ startIndex: 25, type: 'number.ts' },
+			{ startIndex: 26, type: 'delimiter.bracket.ts' },
+			{ startIndex: 27, type: 'string.ts' },
+		]
+	}]
+
+]);

src/typescript/typescript.ts

@@ -0,0 +1,205 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+export const conf: IRichLanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+
+	onEnterRules: [
+		{
+			// e.g. /** | */
+			beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+			afterText: /^\s*\*\/$/,
+			action: { indentAction: _monaco.languages.IndentAction.IndentOutdent, appendText: ' * ' }
+		},
+		{
+			// e.g. /** ...|
+			beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+			action: { indentAction: _monaco.languages.IndentAction.None, appendText: ' * ' }
+		},
+		{
+			// e.g.  * ...|
+			beforeText: /^(\t|(\ \ ))*\ \*(\ ([^\*]|\*(?!\/))*)?$/,
+			action: { indentAction: _monaco.languages.IndentAction.None, appendText: '* ' }
+		},
+		{
+			// e.g.  */|
+			beforeText: /^(\t|(\ \ ))*\ \*\/\s*$/,
+			action: { indentAction: _monaco.languages.IndentAction.None, removeText: 1 }
+		}
+	],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"', notIn: ['string'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '`', close: '`', notIn: ['string', 'comment'] },
+		{ open: "/**", close: " */", notIn: ["string"] }
+	],
+
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*//\\s*#?region\\b"),
+			end: new RegExp("^\\s*//\\s*#?endregion\\b")
+		}
+	}
+};
+
+export const language = {
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.ts',
+
+	keywords: [
+		'abstract', 'as', 'break', 'case', 'catch', 'class', 'continue', 'const',
+		'constructor', 'debugger', 'declare', 'default', 'delete', 'do', 'else',
+		'enum', 'export', 'extends', 'false', 'finally', 'for', 'from', 'function',
+		'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof', 'interface',
+		'is', 'keyof', 'let', 'module', 'namespace', 'never', 'new', 'null', 'package',
+		'private', 'protected', 'public', 'readonly', 'require', 'global', 'return',
+		'set', 'static', 'super', 'switch', 'symbol', 'this', 'throw', 'true', 'try',
+		'type', 'typeof', 'unique', 'var', 'void', 'while', 'with', 'yield', 'async',
+		'await', 'of'
+	],
+
+	typeKeywords: [
+		'any', 'boolean', 'number', 'object', 'string', 'undefined'
+	],
+
+	operators: [
+		'<=', '>=', '==', '!=', '===', '!==', '=>', '+', '-', '**',
+		'*', '/', '%', '++', '--', '<<', '</', '>>', '>>>', '&',
+		'|', '^', '!', '~', '&&', '||', '?', ':', '=', '+=', '-=',
+		'*=', '**=', '/=', '%=', '<<=', '>>=', '>>>=', '&=', '|=',
+		'^=', '@',
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/[{}]/, 'delimiter.bracket'],
+			{ include: 'common' }
+		],
+
+		common: [
+			// identifiers and keywords
+			[/[a-z_$][\w$]*/, {
+				cases: {
+					'@typeKeywords': 'keyword',
+					'@keywords': 'keyword',
+					'@default': 'identifier'
+				}
+			}],
+			// [/[A-Z][\w\$]*/, 'type.identifier' ],  // to show class names nicely
+			[/[A-Z][\w\$]*/, 'identifier'],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, {
+				cases: {
+					'@operators': 'delimiter',
+					'@default': ''
+				}
+			}],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?/, 'number.float'],
+			[/0[xX](@hexdigits)/, 'number.hex'],
+			[/0(@octaldigits)/, 'number.octal'],
+			[/0[bB](@binarydigits)/, 'number.binary'],
+			[/(@digits)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
+			[/"/, 'string', '@string_double'],
+			[/'/, 'string', '@string_single'],
+			[/`/, 'string', '@string_backtick'],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@jsdoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		jsdoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string_double: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+
+		string_single: [
+			[/[^\\']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/'/, 'string', '@pop']
+		],
+
+		string_backtick: [
+			[/\$\{/, { token: 'delimiter.bracket', next: '@bracketCounting' }],
+			[/[^\\`$]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/`/, 'string', '@pop']
+		],
+
+		bracketCounting: [
+			[/\{/, 'delimiter.bracket', '@bracketCounting'],
+			[/\}/, 'delimiter.bracket', '@pop'],
+			{ include: 'common' }
+		],
+	},
+};

test/setup.js

@@ -38,6 +38,7 @@ define(['require'], function (require) {
 			'release/dev/html/html.test',
 			'release/dev/pug/pug.test',
 			'release/dev/java/java.test',
+			'release/dev/javascript/javascript.test',
 			'release/dev/less/less.test',
 			'release/dev/lua/lua.test',
 			'release/dev/markdown/markdown.test',
@@ -45,6 +46,7 @@ define(['require'], function (require) {
 			'release/dev/objective-c/objective-c.test',
 			'release/dev/php/php.test',
 			'release/dev/postiats/postiats.test',
+			'release/dev/powerquery/powerquery.test',
 			'release/dev/powershell/powershell.test',
 			'release/dev/python/python.test',
 			'release/dev/r/r.test',
@@ -66,6 +68,7 @@ define(['require'], function (require) {
 			'release/dev/csp/csp.test',
 			'release/dev/st/st.test',
 			'release/dev/scheme/scheme.test',
+			'release/dev/typescript/typescript.test',
 			'release/dev/clojure/clojure.test',
 			'release/dev/shell/shell.test',
 			'release/dev/perl/perl.test',