Merge remote-tracking branch 'upstream/master'

.editorconfig

@@ -0,0 +1,16 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Tab indentation
+[*]
+indent_style = tab
+indent_size = 4
+trim_trailing_whitespace = true
+
+# The indent size used in the `package.json` file cannot be changed
+# https://github.com/npm/npm/pull/3180#issuecomment-16336516
+[{*.yml,*.yaml,package.json}]
+indent_style = space
+indent_size = 2

.gitignore

@@ -1,2 +1,4 @@
+/.idea/
+/*.iml
 /node_modules/
 /release/

.travis.yml

@@ -1,6 +0,0 @@
-language: node_js
-node_js:
- - "8.9.3"
-script:
- - npm run compile
- - npm run test

.vscode/launch.json

@@ -8,8 +8,8 @@
             "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
             "stopOnEntry": false,
             "args": [
-                "--grep",
-                "php"
+                // "--grep",
+                // "typescript"
             ],
             "cwd": "${workspaceRoot}",
             "preLaunchTask": null,
@@ -25,4 +25,4 @@
             "outDir": null
         }
     ]
-}
+}

.vscode/settings.json

@@ -9,5 +9,6 @@
 	"files.insertFinalNewline": true,
 	"editor.tabSize": 4,
 	"editor.insertSpaces": false,
+	"editor.detectIndentation": false,
 	"typescript.tsdk": "./node_modules/typescript/lib"
 }

README.md

@@ -1,22 +1,35 @@
-# Monaco Languages [![Build Status](https://travis-ci.org/Microsoft/monaco-languages.svg?branch=master)](https://travis-ci.org/Microsoft/monaco-languages)
+# Monaco Languages [![Build Status](https://dev.azure.com/ms/monaco-languages/_apis/build/status/microsoft.monaco-languages?branchName=master)](https://dev.azure.com/ms/monaco-languages/_build/latest?definitionId=140&branchName=master)
 
 Colorization and configuration supports for multiple languages for the Monaco Editor:
 
 ![monaco-languages](https://cloud.githubusercontent.com/assets/5047891/15938606/1fd4bac6-2e74-11e6-8839-d455da8bc8a7.gif)
 
+* apex
+* azcli
 * bat
+* clojure
 * coffee script
 * cpp
 * csharp
+* csp
+* dockerfile
 * fsharp
 * go
+* graphql
 * handlebars
 * html
 * ini
+* java
+* javascript
 * lua
+* markdown
+* msdax
+* mysql
 * objective-c
-* postiats
+* pascal
+* pgsql
 * php
+* postiats
 * powershell
 * pug
 * python
@@ -24,11 +37,16 @@ Colorization and configuration supports for multiple languages for the Monaco Ed
 * razor
 * ruby
 * rust
+* small basic
+* scheme
+* solidity
 * sql
+* st
 * swift
+* typescript
 * vb
 * xml
-* small basic
+* yaml
 
 Also `css` dialects:
 
@@ -49,7 +67,7 @@ This npm module is bundled and distributed in the [monaco-editor](https://www.np
 * initial setup with `npm install .`
 * compile with `npm run watch`
 * test with `npm run test`
-* bundle with `npm run prepublish`
+* bundle with `npm run prepublishOnly`
 
 ## Dev: Adding a new language
 

azure-pipelines.yml

@@ -0,0 +1,28 @@
+# Node.js
+# Build a general Node.js project with npm.
+# Add steps that analyze code, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/javascript
+
+trigger:
+- master
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+steps:
+- task: NodeTool@0
+  inputs:
+    versionSpec: '10.x'
+  displayName: 'Install Node.js'
+
+- script: |
+    npm install
+  displayName: 'npm install'
+
+- script: |
+    npm run compile
+  displayName: 'compile'
+
+- script: |
+    npm test
+  displayName: 'test'

package.json

@@ -1,12 +1,12 @@
 {
   "name": "monaco-languages",
-  "version": "1.2.0",
+  "version": "1.7.0",
   "description": "Bundle of many languages for the Monaco Editor.",
   "scripts": {
     "compile": "mrmdir ./release && tsc -p ./src/tsconfig.json && tsc -p ./src/tsconfig.esm.json",
     "watch": "tsc -p ./src --watch",
     "test": "mocha",
-    "prepublish": "npm run compile && node ./scripts/bundle"
+    "prepublishOnly": "npm run compile && node ./scripts/bundle"
   },
   "author": "Microsoft Corporation",
   "license": "MIT",
@@ -18,12 +18,12 @@
     "url": "https://github.com/Microsoft/monaco-languages/issues"
   },
   "devDependencies": {
-    "jsdom-no-contextify": "^3.1.0",
-    "mocha": "^3.4.2",
-    "monaco-editor-core": "0.12.0",
+    "jsdom": "^15.1.1",
+    "mocha": "^6.1.4",
+    "monaco-editor-core": "0.17.0",
     "monaco-plugin-helpers": "^1.0.2",
-    "requirejs": "^2.3.5",
-    "typescript": "2.7.2",
-    "uglify-js": "^3.3.14"
+    "requirejs": "^2.3.6",
+    "typescript": "3.5.3",
+    "uglify-js": "^3.6.0"
   }
 }

scripts/bundle.js

@@ -34,11 +34,14 @@ bundleOne('html/html');
 bundleOne('ini/ini');
 bundleOne('pug/pug');
 bundleOne('java/java');
+bundleOne('javascript/javascript');
+bundleOne('kotlin/kotlin');
 bundleOne('less/less');
 bundleOne('lua/lua');
 bundleOne('markdown/markdown');
 bundleOne('msdax/msdax');
 bundleOne('objective-c/objective-c');
+bundleOne('pascal/pascal');
 bundleOne('php/php');
 bundleOne('powershell/powershell');
 bundleOne('postiats/postiats');
@@ -51,6 +54,7 @@ bundleOne('scss/scss');
 bundleOne('sql/sql');
 bundleOne('st/st');
 bundleOne('swift/swift');
+bundleOne('typescript/typescript');
 bundleOne('vb/vb');
 bundleOne('xml/xml');
 bundleOne('yaml/yaml');
@@ -61,6 +65,15 @@ bundleOne('redshift/redshift');
 bundleOne('pgsql/pgsql');
 bundleOne('redis/redis');
 bundleOne('csp/csp');
+bundleOne('scheme/scheme');
+bundleOne('clojure/clojure');
+bundleOne('shell/shell');
+bundleOne('perl/perl');
+bundleOne('powerquery/powerquery');
+bundleOne('azcli/azcli');
+bundleOne('apex/apex');
+bundleOne('tcl/tcl');
+bundleOne('graphql/graphql');
 
 function bundleOne(moduleId, exclude) {
 	requirejs.optimize({
@@ -72,7 +85,7 @@ function bundleOne(moduleId, exclude) {
 			'vs/basic-languages': REPO_ROOT + '/release/dev'
 		},
 		optimize: 'none'
-	}, function(buildResponse) {
+	}, function (buildResponse) {
 		const filePath = path.join(REPO_ROOT, 'release/min/' + moduleId + '.js');
 		const fileContents = fs.readFileSync(filePath).toString();
 		console.log();

src/_.contribution.ts

@@ -8,7 +8,7 @@
 const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
 
 interface ILang extends monaco.languages.ILanguageExtensionPoint {
-	loader: () => monaco.Promise<ILangImpl>;
+	loader: () => Promise<ILangImpl>;
 }
 
 interface ILangImpl {
@@ -18,7 +18,7 @@ interface ILangImpl {
 
 let languageDefinitions: { [languageId: string]: ILang } = {};
 
-function _loadLanguage(languageId: string): monaco.Promise<void> {
+function _loadLanguage(languageId: string): Promise<void> {
 	const loader = languageDefinitions[languageId].loader;
 	return loader().then((mod) => {
 		_monaco.languages.setMonarchTokensProvider(languageId, mod.language);
@@ -26,9 +26,9 @@ function _loadLanguage(languageId: string): monaco.Promise<void> {
 	});
 }
 
-let languagePromises: { [languageId: string]: monaco.Promise<void> } = {};
+let languagePromises: { [languageId: string]: Promise<void> } = {};
 
-export function loadLanguage(languageId: string): monaco.Promise<void> {
+export function loadLanguage(languageId: string): Promise<void> {
 	if (!languagePromises[languageId]) {
 		languagePromises[languageId] = _loadLanguage(languageId);
 	}

src/apex/apex.contribution.ts

@@ -0,0 +1,15 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'apex',
+	extensions: ['.cls'],
+	aliases: ['Apex', 'apex'],
+	mimetypes: ['text/x-apex-source', 'text/x-apex'],
+	loader: () => import('./apex')
+});

src/apex/apex.test.ts

@@ -0,0 +1,651 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('apex', [
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.apex' }
+		]
+	}],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[{
+		line: '/* //*/ a',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}],
+
+	[{
+		line: '/almost a comment',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.apex' },
+			{ startIndex: 1, type: 'identifier.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.apex' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.apex' },
+			{ startIndex: 5, type: 'delimiter.apex' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.apex' }
+		]
+	}],
+
+	[{
+		line: 'int x = 1; // my comment // is a nice one',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.int.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.apex' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.apex' },
+			{ startIndex: 9, type: 'delimiter.apex' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'comment.apex' }
+		]
+	}],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}],
+
+	[{
+		line: 'int x = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.int.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.apex' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.apex' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.apex' },
+			{ startIndex: 32, type: 'delimiter.apex' }
+		]
+	}],
+
+	[{
+		line: 'int x = /* comment */ 1; */',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.int.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.apex' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.apex' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.apex' },
+			{ startIndex: 23, type: 'delimiter.apex' },
+			{ startIndex: 24, type: '' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.apex' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.apex' },
+			{ startIndex: 8, type: 'delimiter.apex' }
+		]
+	}],
+
+	[{
+		line: 'x = /*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.apex' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.apex' }
+		]
+	}],
+
+	// Comments - range comment, multiple lines
+	[{
+		line: '/* start of multiline comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}, {
+		line: 'a comment between without a star',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}, {
+		line: 'end of multiline comment*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}],
+
+	[{
+		line: 'int x = /* start a comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.int.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.apex' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.apex' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' }
+		]
+	}, {
+		line: 'and end it */ 2;',
+		tokens: [
+			{ startIndex: 0, type: 'comment.apex' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'number.apex' },
+			{ startIndex: 15, type: 'delimiter.apex' }
+		]
+	}],
+
+	// Comments - apex doc, multiple lines
+	[{
+		line: '/** start of Apex Doc',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.apex' }
+		]
+	}, {
+		line: 'a comment between without a star',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.apex' }
+		]
+	}, {
+		line: 'end of multiline comment*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.apex' }
+		]
+	}],
+
+	// Keywords
+	[{
+		line: 'package test; class Program { static void main(String[] args) {} } }',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.package.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.apex' },
+			{ startIndex: 12, type: 'delimiter.apex' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.class.apex' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'type.identifier.apex' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 28, type: 'delimiter.curly.apex' },
+			{ startIndex: 29, type: '' },
+			{ startIndex: 30, type: 'keyword.static.apex' },
+			{ startIndex: 36, type: '' },
+			{ startIndex: 37, type: 'keyword.void.apex' },
+			{ startIndex: 41, type: '' },
+			{ startIndex: 42, type: 'identifier.apex' },
+			{ startIndex: 46, type: 'delimiter.parenthesis.apex' },
+			{ startIndex: 47, type: 'type.identifier.apex' },
+			{ startIndex: 53, type: 'delimiter.square.apex' },
+			{ startIndex: 55, type: '' },
+			{ startIndex: 56, type: 'identifier.apex' },
+			{ startIndex: 60, type: 'delimiter.parenthesis.apex' },
+			{ startIndex: 61, type: '' },
+			{ startIndex: 62, type: 'delimiter.curly.apex' },
+			{ startIndex: 64, type: '' },
+			{ startIndex: 65, type: 'delimiter.curly.apex' },
+			{ startIndex: 66, type: '' },
+			{ startIndex: 67, type: 'delimiter.curly.apex' }
+		]
+	}],
+
+	// Keywords with case variations
+	[{
+		line: 'Package test; CLASS Program { Static void main(String[] args) {} } }',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.Package.apex' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.apex' },
+			{ startIndex: 12, type: 'delimiter.apex' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.CLASS.apex' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'type.identifier.apex' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 28, type: 'delimiter.curly.apex' },
+			{ startIndex: 29, type: '' },
+			{ startIndex: 30, type: 'keyword.Static.apex' },
+			{ startIndex: 36, type: '' },
+			{ startIndex: 37, type: 'keyword.void.apex' },
+			{ startIndex: 41, type: '' },
+			{ startIndex: 42, type: 'identifier.apex' },
+			{ startIndex: 46, type: 'delimiter.parenthesis.apex' },
+			{ startIndex: 47, type: 'type.identifier.apex' },
+			{ startIndex: 53, type: 'delimiter.square.apex' },
+			{ startIndex: 55, type: '' },
+			{ startIndex: 56, type: 'identifier.apex' },
+			{ startIndex: 60, type: 'delimiter.parenthesis.apex' },
+			{ startIndex: 61, type: '' },
+			{ startIndex: 62, type: 'delimiter.curly.apex' },
+			{ startIndex: 64, type: '' },
+			{ startIndex: 65, type: 'delimiter.curly.apex' },
+			{ startIndex: 66, type: '' },
+			{ startIndex: 67, type: 'delimiter.curly.apex' }
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '0.10',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '10e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '10f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5e-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5E3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5E-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5F',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '1.72E3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '1.72E3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '1.72E-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '1.72e3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '1.72e3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '1.72e-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '23L',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '23l',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '0_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '5_______2',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '3_.1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: 'identifier.apex' },
+			{ startIndex: 2, type: 'delimiter.apex' },
+			{ startIndex: 3, type: 'number.float.apex' }
+		]
+	}],
+
+	[{
+		line: '3._1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: 'delimiter.apex' },
+			{ startIndex: 2, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '999_99_9999_L',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 11, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 2, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '0_x52',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '0x_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: 'identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '23.5L',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.apex' },
+			{ startIndex: 4, type: 'type.identifier.apex' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: 'delimiter.apex' },
+			{ startIndex: 2, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 3, type: 'delimiter.apex' },
+			{ startIndex: 4, type: 'number.apex' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.apex' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.apex' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.apex' }
+		]
+	}],
+
+	// single line Strings
+	[{
+		line: 'String s = "I\'m an Apex String";',
+		tokens: [
+			{ startIndex: 0, type: 'type.identifier.apex' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'identifier.apex' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'delimiter.apex' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'string.apex' },
+			{ startIndex: 31, type: 'delimiter.apex' }
+		]
+	}],
+
+	[{
+		line: 'String s = "concatenated" + " String" ;',
+		tokens: [
+			{ startIndex: 0, type: 'type.identifier.apex' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'identifier.apex' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'delimiter.apex' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'string.apex' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'delimiter.apex' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 28, type: 'string.apex' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'delimiter.apex' }
+		]
+	}],
+
+	[{
+		line: '"quote in a string"',
+		tokens: [
+			{ startIndex: 0, type: 'string.apex' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.apex' },
+			{ startIndex: 10, type: 'string.escape.apex' },
+			{ startIndex: 12, type: 'string.apex' },
+			{ startIndex: 18, type: 'string.escape.apex' },
+			{ startIndex: 20, type: 'string.apex' }
+		]
+	}],
+
+	[{
+		line: '"\\"',
+		tokens: [
+			{ startIndex: 0, type: 'string.invalid.apex' }
+		]
+	}],
+
+	// Annotations
+	[{
+		line: '@',
+		tokens: [
+			{ startIndex: 0, type: '' }
+		]
+	}],
+
+	[{
+		line: '@Override',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.apex' }
+		]
+	}],
+
+	[{
+		line: '@SuppressWarnings(value = "aString")',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.apex' },
+			{ startIndex: 17, type: 'delimiter.parenthesis.apex' },
+			{ startIndex: 18, type: 'identifier.apex' },
+			{ startIndex: 23, type: '' },
+			{ startIndex: 24, type: 'delimiter.apex' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'string.apex' },
+			{ startIndex: 35, type: 'delimiter.parenthesis.apex' }
+		]
+	}],
+
+	[{
+		line: '@ AnnotationWithKeywordAfter private',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.apex' },
+			{ startIndex: 28, type: '' },
+			{ startIndex: 29, type: 'keyword.private.apex' }
+		]
+	}]
+]);
+

src/apex/apex.ts

@@ -0,0 +1,310 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')'],
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+		{ open: '<', close: '>' },
+	],
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*//\\s*(?:(?:#?region\\b)|(?:<editor-fold\\b))"),
+			end: new RegExp("^\\s*//\\s*(?:(?:#?endregion\\b)|(?:</editor-fold>))")
+		}
+	}
+};
+
+const keywords = [
+	'abstract',
+	'activate',
+	'and',
+	'any',
+	'array',
+	'as',
+	'asc',
+	'assert',
+	'autonomous',
+	'begin',
+	'bigdecimal',
+	'blob',
+	'boolean',
+	'break',
+	'bulk',
+	'by',
+	'case',
+	'cast',
+	'catch',
+	'char',
+	'class',
+	'collect',
+	'commit',
+	'const',
+	'continue',
+	'convertcurrency',
+	'decimal',
+	'default',
+	'delete',
+	'desc',
+	'do',
+	'double',
+	'else',
+	'end',
+	'enum',
+	'exception',
+	'exit',
+	'export',
+	'extends',
+	'false',
+	'final',
+	'finally',
+	'float',
+	'for',
+	'from',
+	'future',
+	'get',
+	'global',
+	'goto',
+	'group',
+	'having',
+	'hint',
+	'if',
+	'implements',
+	'import',
+	'in',
+	'inner',
+	'insert',
+	'instanceof',
+	'int',
+	'interface',
+	'into',
+	'join',
+	'last_90_days',
+	'last_month',
+	'last_n_days',
+	'last_week',
+	'like',
+	'limit',
+	'list',
+	'long',
+	'loop',
+	'map',
+	'merge',
+	'native',
+	'new',
+	'next_90_days',
+	'next_month',
+	'next_n_days',
+	'next_week',
+	'not',
+	'null',
+	'nulls',
+	'number',
+	'object',
+	'of',
+	'on',
+	'or',
+	'outer',
+	'override',
+	'package',
+	'parallel',
+	'pragma',
+	'private',
+	'protected',
+	'public',
+	'retrieve',
+	'return',
+	'returning',
+	'rollback',
+	'savepoint',
+	'search',
+	'select',
+	'set',
+	'short',
+	'sort',
+	'stat',
+	'static',
+	'strictfp',
+	'super',
+	'switch',
+	'synchronized',
+	'system',
+	'testmethod',
+	'then',
+	'this',
+	'this_month',
+	'this_week',
+	'throw',
+	'throws',
+	'today',
+	'tolabel',
+	'tomorrow',
+	'transaction',
+	'transient',
+	'trigger',
+	'true',
+	'try',
+	'type',
+	'undelete',
+	'update',
+	'upsert',
+	'using',
+	'virtual',
+	'void',
+	'volatile',
+	'webservice',
+	'when',
+	'where',
+	'while',
+	'yesterday'
+];
+
+// create case variations of the keywords - apex is case insensitive, but we can't make the highlighter case insensitive
+// because we use a heuristic to assume that identifiers starting with an upper case letter are types.
+const uppercaseFirstLetter = (lowercase) => lowercase.charAt(0).toUpperCase() + lowercase.substr(1);
+
+let keywordsWithCaseVariations = [];
+keywords.forEach(lowercase => {
+	keywordsWithCaseVariations.push(lowercase);
+	keywordsWithCaseVariations.push(lowercase.toUpperCase());
+	keywordsWithCaseVariations.push(uppercaseFirstLetter(lowercase));
+})
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.apex',
+
+	keywords: keywordsWithCaseVariations,
+
+	operators: [
+		'=', '>', '<', '!', '~', '?', ':',
+		'==', '<=', '>=', '!=', '&&', '||', '++', '--',
+		'+', '-', '*', '/', '&', '|', '^', '%', '<<',
+		'>>', '>>>', '+=', '-=', '*=', '/=', '&=', '|=',
+		'^=', '%=', '<<=', '>>=', '>>>='
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-z_$][\w$]*/, {
+				cases: {
+					'@keywords': { token: 'keyword.$0' },
+					'@default': 'identifier'
+				}
+			}],
+
+			// assume that identifiers starting with an uppercase letter are types
+			[/[A-Z][\w\$]*/, {
+				cases: {
+					'@keywords': { token: 'keyword.$0' },
+					'@default': 'type.identifier'
+				}
+			}],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, {
+				cases: {
+					'@operators': 'delimiter',
+					'@default': ''
+				}
+			}],
+
+			// @ annotations.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, 'annotation'],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)[fFdD]/, 'number.float'],
+			[/(@digits)[lL]?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string."' ],
+			[/'/,  'string', '@string.\'' ],
+
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@apexdoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		//Identical copy of comment above, except for the addition of .doc
+		apexdoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': 'string' }} ]
+		],
+	},
+};

src/azcli/azcli.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'azcli',
+	extensions: ['.azcli'],
+	aliases: ['Azure CLI', 'azcli'],
+	loader: () => import('./azcli')
+});

src/azcli/azcli.test.ts

@@ -0,0 +1,157 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization as actualTestTokenization, ITestItem } from '../test/testRunner';
+
+function testTokenization(_language: string | string[], tests: ITestItem[][]): void {
+	tests = tests.map(t => {
+		return t.map(t => {
+			return {
+				line: t.line.replace(/\n/g, ' '),
+				tokens: t.tokens
+			};
+		});
+	});
+	actualTestTokenization(_language, tests);
+}
+
+testTokenization('azcli', [
+	// Comment single line
+	[{
+		line: '#',
+		tokens: [
+			{ startIndex: 0, type: 'comment.azcli' }
+		]
+	}],
+	[{
+		line: '# az find -q secret',
+		tokens: [
+			{ startIndex: 0, type: 'comment.azcli' }
+		]
+	}],
+	[{
+		line: '    # az find -q secret',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' },
+			{ startIndex: 4, type: 'comment.azcli' }
+		]
+	}],
+	[{
+		line: '#az find -q secret',
+		tokens: [
+			{ startIndex: 0, type: 'comment.azcli' }
+		]
+	}],
+
+	// Other cases
+	[{
+		line: 'az find -q secret',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' },
+			{ startIndex: 7, type: 'key.identifier.azcli' },
+			{ startIndex: 11, type: 'string.azcli' }
+		]
+	}],
+	[{
+		line: '',
+		tokens: [
+		]
+	}],
+	[{
+		line: ' ',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' }
+		]
+	}],
+	[{
+		line: '--assignee',
+		tokens: [
+			{ startIndex: 0, type: 'key.identifier.azcli' }
+		]
+	}],
+	[{
+		line: '    --service-principal',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' },
+			{ startIndex: 3, type: 'key.identifier.azcli' }
+		]
+	}],
+	[{
+		line: 'az ad sp create-for-rb  --name ServicePrincipalName --password PASSWORD',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' },
+			{ startIndex: 23, type: 'key.identifier.azcli' },
+			{ startIndex: 31, type: 'string.azcli' },
+			{ startIndex: 52, type: 'key.identifier.azcli' },
+			{ startIndex: 63, type: 'string.azcli' }
+		]
+	}],
+	[{
+		line: '--name!~`\"$%^&*(|\/\.,-=+',
+		tokens: [
+			{ startIndex: 0, type: 'key.identifier.azcli' }
+		]
+	}],
+	[{
+		line: '--name#some comment',
+		tokens: [
+			{ startIndex: 0, type: 'key.identifier.azcli' },
+			{ startIndex: 6, type: 'comment.azcli' }
+		]
+	}],
+	[{
+		line: '--query osPro ``````',
+		tokens: [
+			{ startIndex: 0, type: 'key.identifier.azcli' },
+			{ startIndex: 8, type: 'string.azcli' }
+		]
+	}],
+	[{
+		line: 'az ad sp create-for-rbac',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' }
+		]
+	}],
+	[{
+		line: '123456789',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' }
+		]
+	}],
+	[{
+		line: '- abc',
+		tokens: [
+			{ startIndex: 0, type: 'key.identifier.azcli' },
+			{ startIndex: 2, type: 'string.azcli' }
+		]
+	}],
+	[{
+		line: '- @!$()',
+		tokens: [
+			{ startIndex: 0, type: 'key.identifier.azcli' },
+			{ startIndex: 2, type: 'string.azcli' }
+		]
+	}],
+	[{
+		line: '""',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' }
+		]
+	}],
+	[{
+		line: '// some text',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' }
+		]
+	}],
+	[{
+		line: `'APP_ID'`,
+		tokens: [
+			{ startIndex: 0, type: 'keyword.azcli' }
+		]
+	}]
+]);

src/azcli/azcli.ts

@@ -0,0 +1,65 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+	}
+};
+
+export const language = <ILanguage>{
+	defaultToken: 'keyword',
+	ignoreCase: true,
+	tokenPostfix: '.azcli',
+
+	str: /[^#\s]/,
+
+	tokenizer: {
+		root: [
+			{include: '@comment'},
+			[/\s-+@str*\s*/, {
+				cases: {
+				  '@eos': { token: 'key.identifier', next: '@popall' },
+				  '@default': { token: 'key.identifier', next: '@type' }
+				}
+			}],
+			[/^-+@str*\s*/, {
+				cases: {
+				  '@eos': { token: 'key.identifier', next: '@popall' },
+				  '@default': { token: 'key.identifier', next: '@type' }
+				}
+			}]
+		],
+
+		type: [
+			{include: '@comment'},
+			[/-+@str*\s*/, {
+				cases: {
+					'@eos': { token: 'key.identifier', next: '@popall' },
+					'@default': 'key.identifier'
+				}
+			}],
+			[/@str+\s*/, {
+				cases: {
+					'@eos': { token: 'string', next: '@popall' },
+					'@default': 'string'
+			  	}
+			}]
+		],
+
+		comment: [
+			[/#.*$/, {
+				cases: {
+					'@eos': { token: 'comment', next: '@popall' }
+				}
+			}]
+		]
+	}
+};

src/bat/bat.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'bat',
 	extensions: ['.bat', '.cmd'],
 	aliases: ['Batch', 'bat'],
-	loader: () => _monaco.Promise.wrap(import('./bat'))
+	loader: () => import('./bat')
 });

src/clojure/clojure.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'clojure',
+	extensions: ['.clj', '.cljs', '.cljc', '.edn'],
+	aliases: ['clojure', 'Clojure'],
+	loader: () => import('./clojure')
+});

src/clojure/clojure.test.ts

@@ -0,0 +1,901 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {ITestItem, testTokenization} from '../test/testRunner';
+
+const specialForms = [
+	'.',
+	'catch',
+	'def',
+	'do',
+	'if',
+	'monitor-enter',
+	'monitor-exit',
+	'new',
+	'quote',
+	'recur',
+	'set!',
+	'throw',
+	'try',
+	'var',
+];
+
+const coreSymbols = [
+	'*',
+	'*\'',
+	'*1',
+	'*2',
+	'*3',
+	'*agent*',
+	'*allow-unresolved-vars*',
+	'*assert*',
+	'*clojure-version*',
+	'*command-line-args*',
+	'*compile-files*',
+	'*compile-path*',
+	'*compiler-options*',
+	'*data-readers*',
+	'*default-data-reader-fn*',
+	'*e',
+	'*err*',
+	'*file*',
+	'*flush-on-newline*',
+	'*fn-loader*',
+	'*in*',
+	'*math-context*',
+	'*ns*',
+	'*out*',
+	'*print-dup*',
+	'*print-length*',
+	'*print-level*',
+	'*print-meta*',
+	'*print-namespace-maps*',
+	'*print-readably*',
+	'*read-eval*',
+	'*reader-resolver*',
+	'*source-path*',
+	'*suppress-read*',
+	'*unchecked-math*',
+	'*use-context-classloader*',
+	'*verbose-defrecords*',
+	'*warn-on-reflection*',
+	'+',
+	'+\'',
+	'-',
+	'-\'',
+	'->',
+	'->>',
+	'->ArrayChunk',
+	'->Eduction',
+	'->Vec',
+	'->VecNode',
+	'->VecSeq',
+	'-cache-protocol-fn',
+	'-reset-methods',
+	'..',
+	'/',
+	'<',
+	'<=',
+	'=',
+	'==',
+	'>',
+	'>=',
+	'EMPTY-NODE',
+	'Inst',
+	'StackTraceElement->vec',
+	'Throwable->map',
+	'accessor',
+	'aclone',
+	'add-classpath',
+	'add-watch',
+	'agent',
+	'agent-error',
+	'agent-errors',
+	'aget',
+	'alength',
+	'alias',
+	'all-ns',
+	'alter',
+	'alter-meta!',
+	'alter-var-root',
+	'amap',
+	'ancestors',
+	'and',
+	'any?',
+	'apply',
+	'areduce',
+	'array-map',
+	'as->',
+	'aset',
+	'aset-boolean',
+	'aset-byte',
+	'aset-char',
+	'aset-double',
+	'aset-float',
+	'aset-int',
+	'aset-long',
+	'aset-short',
+	'assert',
+	'assoc',
+	'assoc!',
+	'assoc-in',
+	'associative?',
+	'atom',
+	'await',
+	'await-for',
+	'await1',
+	'bases',
+	'bean',
+	'bigdec',
+	'bigint',
+	'biginteger',
+	'binding',
+	'bit-and',
+	'bit-and-not',
+	'bit-clear',
+	'bit-flip',
+	'bit-not',
+	'bit-or',
+	'bit-set',
+	'bit-shift-left',
+	'bit-shift-right',
+	'bit-test',
+	'bit-xor',
+	'boolean',
+	'boolean-array',
+	'boolean?',
+	'booleans',
+	'bound-fn',
+	'bound-fn*',
+	'bound?',
+	'bounded-count',
+	'butlast',
+	'byte',
+	'byte-array',
+	'bytes',
+	'bytes?',
+	'case',
+	'cast',
+	'cat',
+	'char',
+	'char-array',
+	'char-escape-string',
+	'char-name-string',
+	'char?',
+	'chars',
+	'chunk',
+	'chunk-append',
+	'chunk-buffer',
+	'chunk-cons',
+	'chunk-first',
+	'chunk-next',
+	'chunk-rest',
+	'chunked-seq?',
+	'class',
+	'class?',
+	'clear-agent-errors',
+	'clojure-version',
+	'coll?',
+	'comment',
+	'commute',
+	'comp',
+	'comparator',
+	'compare',
+	'compare-and-set!',
+	'compile',
+	'complement',
+	'completing',
+	'concat',
+	'cond',
+	'cond->',
+	'cond->>',
+	'condp',
+	'conj',
+	'conj!',
+	'cons',
+	'constantly',
+	'construct-proxy',
+	'contains?',
+	'count',
+	'counted?',
+	'create-ns',
+	'create-struct',
+	'cycle',
+	'dec',
+	'dec\'',
+	'decimal?',
+	'declare',
+	'dedupe',
+	'default-data-readers',
+	'definline',
+	'definterface',
+	'defmacro',
+	'defmethod',
+	'defmulti',
+	'defn',
+	'defn-',
+	'defonce',
+	'defprotocol',
+	'defrecord',
+	'defstruct',
+	'deftype',
+	'delay',
+	'delay?',
+	'deliver',
+	'denominator',
+	'deref',
+	'derive',
+	'descendants',
+	'destructure',
+	'disj',
+	'disj!',
+	'dissoc',
+	'dissoc!',
+	'distinct',
+	'distinct?',
+	'doall',
+	'dorun',
+	'doseq',
+	'dosync',
+	'dotimes',
+	'doto',
+	'double',
+	'double-array',
+	'double?',
+	'doubles',
+	'drop',
+	'drop-last',
+	'drop-while',
+	'eduction',
+	'empty',
+	'empty?',
+	'ensure',
+	'ensure-reduced',
+	'enumeration-seq',
+	'error-handler',
+	'error-mode',
+	'eval',
+	'even?',
+	'every-pred',
+	'every?',
+	'ex-data',
+	'ex-info',
+	'extend',
+	'extend-protocol',
+	'extend-type',
+	'extenders',
+	'extends?',
+	'false?',
+	'ffirst',
+	'file-seq',
+	'filter',
+	'filterv',
+	'find',
+	'find-keyword',
+	'find-ns',
+	'find-protocol-impl',
+	'find-protocol-method',
+	'find-var',
+	'first',
+	'flatten',
+	'float',
+	'float-array',
+	'float?',
+	'floats',
+	'flush',
+	'fn',
+	'fn?',
+	'fnext',
+	'fnil',
+	'for',
+	'force',
+	'format',
+	'frequencies',
+	'future',
+	'future-call',
+	'future-cancel',
+	'future-cancelled?',
+	'future-done?',
+	'future?',
+	'gen-class',
+	'gen-interface',
+	'gensym',
+	'get',
+	'get-in',
+	'get-method',
+	'get-proxy-class',
+	'get-thread-bindings',
+	'get-validator',
+	'group-by',
+	'halt-when',
+	'hash',
+	'hash-combine',
+	'hash-map',
+	'hash-ordered-coll',
+	'hash-set',
+	'hash-unordered-coll',
+	'ident?',
+	'identical?',
+	'identity',
+	'if-let',
+	'if-not',
+	'if-some',
+	'ifn?',
+	'import',
+	'in-ns',
+	'inc',
+	'inc\'',
+	'indexed?',
+	'init-proxy',
+	'inst-ms',
+	'inst-ms*',
+	'inst?',
+	'instance?',
+	'int',
+	'int-array',
+	'int?',
+	'integer?',
+	'interleave',
+	'intern',
+	'interpose',
+	'into',
+	'into-array',
+	'ints',
+	'io!',
+	'isa?',
+	'iterate',
+	'iterator-seq',
+	'juxt',
+	'keep',
+	'keep-indexed',
+	'key',
+	'keys',
+	'keyword',
+	'keyword?',
+	'last',
+	'lazy-cat',
+	'lazy-seq',
+	'let',
+	'letfn',
+	'line-seq',
+	'list',
+	'list*',
+	'list?',
+	'load',
+	'load-file',
+	'load-reader',
+	'load-string',
+	'loaded-libs',
+	'locking',
+	'long',
+	'long-array',
+	'longs',
+	'loop',
+	'macroexpand',
+	'macroexpand-1',
+	'make-array',
+	'make-hierarchy',
+	'map',
+	'map-entry?',
+	'map-indexed',
+	'map?',
+	'mapcat',
+	'mapv',
+	'max',
+	'max-key',
+	'memfn',
+	'memoize',
+	'merge',
+	'merge-with',
+	'meta',
+	'method-sig',
+	'methods',
+	'min',
+	'min-key',
+	'mix-collection-hash',
+	'mod',
+	'munge',
+	'name',
+	'namespace',
+	'namespace-munge',
+	'nat-int?',
+	'neg-int?',
+	'neg?',
+	'newline',
+	'next',
+	'nfirst',
+	'nil?',
+	'nnext',
+	'not',
+	'not-any?',
+	'not-empty',
+	'not-every?',
+	'not=',
+	'ns',
+	'ns-aliases',
+	'ns-imports',
+	'ns-interns',
+	'ns-map',
+	'ns-name',
+	'ns-publics',
+	'ns-refers',
+	'ns-resolve',
+	'ns-unalias',
+	'ns-unmap',
+	'nth',
+	'nthnext',
+	'nthrest',
+	'num',
+	'number?',
+	'numerator',
+	'object-array',
+	'odd?',
+	'or',
+	'parents',
+	'partial',
+	'partition',
+	'partition-all',
+	'partition-by',
+	'pcalls',
+	'peek',
+	'persistent!',
+	'pmap',
+	'pop',
+	'pop!',
+	'pop-thread-bindings',
+	'pos-int?',
+	'pos?',
+	'pr',
+	'pr-str',
+	'prefer-method',
+	'prefers',
+	'primitives-classnames',
+	'print',
+	'print-ctor',
+	'print-dup',
+	'print-method',
+	'print-simple',
+	'print-str',
+	'printf',
+	'println',
+	'println-str',
+	'prn',
+	'prn-str',
+	'promise',
+	'proxy',
+	'proxy-call-with-super',
+	'proxy-mappings',
+	'proxy-name',
+	'proxy-super',
+	'push-thread-bindings',
+	'pvalues',
+	'qualified-ident?',
+	'qualified-keyword?',
+	'qualified-symbol?',
+	'quot',
+	'rand',
+	'rand-int',
+	'rand-nth',
+	'random-sample',
+	'range',
+	'ratio?',
+	'rational?',
+	'rationalize',
+	're-find',
+	're-groups',
+	're-matcher',
+	're-matches',
+	're-pattern',
+	're-seq',
+	'read',
+	'read-line',
+	'read-string',
+	'reader-conditional',
+	'reader-conditional?',
+	'realized?',
+	'record?',
+	'reduce',
+	'reduce-kv',
+	'reduced',
+	'reduced?',
+	'reductions',
+	'ref',
+	'ref-history-count',
+	'ref-max-history',
+	'ref-min-history',
+	'ref-set',
+	'refer',
+	'refer-clojure',
+	'reify',
+	'release-pending-sends',
+	'rem',
+	'remove',
+	'remove-all-methods',
+	'remove-method',
+	'remove-ns',
+	'remove-watch',
+	'repeat',
+	'repeatedly',
+	'replace',
+	'replicate',
+	'require',
+	'reset!',
+	'reset-meta!',
+	'reset-vals!',
+	'resolve',
+	'rest',
+	'restart-agent',
+	'resultset-seq',
+	'reverse',
+	'reversible?',
+	'rseq',
+	'rsubseq',
+	'run!',
+	'satisfies?',
+	'second',
+	'select-keys',
+	'send',
+	'send-off',
+	'send-via',
+	'seq',
+	'seq?',
+	'seqable?',
+	'seque',
+	'sequence',
+	'sequential?',
+	'set',
+	'set-agent-send-executor!',
+	'set-agent-send-off-executor!',
+	'set-error-handler!',
+	'set-error-mode!',
+	'set-validator!',
+	'set?',
+	'short',
+	'short-array',
+	'shorts',
+	'shuffle',
+	'shutdown-agents',
+	'simple-ident?',
+	'simple-keyword?',
+	'simple-symbol?',
+	'slurp',
+	'some',
+	'some->',
+	'some->>',
+	'some-fn',
+	'some?',
+	'sort',
+	'sort-by',
+	'sorted-map',
+	'sorted-map-by',
+	'sorted-set',
+	'sorted-set-by',
+	'sorted?',
+	'special-symbol?',
+	'spit',
+	'split-at',
+	'split-with',
+	'str',
+	'string?',
+	'struct',
+	'struct-map',
+	'subs',
+	'subseq',
+	'subvec',
+	'supers',
+	'swap!',
+	'swap-vals!',
+	'symbol',
+	'symbol?',
+	'sync',
+	'tagged-literal',
+	'tagged-literal?',
+	'take',
+	'take-last',
+	'take-nth',
+	'take-while',
+	'test',
+	'the-ns',
+	'thread-bound?',
+	'time',
+	'to-array',
+	'to-array-2d',
+	'trampoline',
+	'transduce',
+	'transient',
+	'tree-seq',
+	'true?',
+	'type',
+	'unchecked-add',
+	'unchecked-add-int',
+	'unchecked-byte',
+	'unchecked-char',
+	'unchecked-dec',
+	'unchecked-dec-int',
+	'unchecked-divide-int',
+	'unchecked-double',
+	'unchecked-float',
+	'unchecked-inc',
+	'unchecked-inc-int',
+	'unchecked-int',
+	'unchecked-long',
+	'unchecked-multiply',
+	'unchecked-multiply-int',
+	'unchecked-negate',
+	'unchecked-negate-int',
+	'unchecked-remainder-int',
+	'unchecked-short',
+	'unchecked-subtract',
+	'unchecked-subtract-int',
+	'underive',
+	'unquote',
+	'unquote-splicing',
+	'unreduced',
+	'unsigned-bit-shift-right',
+	'update',
+	'update-in',
+	'update-proxy',
+	'uri?',
+	'use',
+	'uuid?',
+	'val',
+	'vals',
+	'var-get',
+	'var-set',
+	'var?',
+	'vary-meta',
+	'vec',
+	'vector',
+	'vector-of',
+	'vector?',
+	'volatile!',
+	'volatile?',
+	'vreset!',
+	'vswap!',
+	'when',
+	'when-first',
+	'when-let',
+	'when-not',
+	'when-some',
+	'while',
+	'with-bindings',
+	'with-bindings*',
+	'with-in-str',
+	'with-loading-context',
+	'with-local-vars',
+	'with-meta',
+	'with-open',
+	'with-out-str',
+	'with-precision',
+	'with-redefs',
+	'with-redefs-fn',
+	'xml-seq',
+	'zero?',
+	'zipmap',
+];
+
+function createTestCases(specialForms: string[], type: string): ITestItem[] {
+	const testCases = [];
+
+	for (const specialForm of specialForms) {
+		testCases.push({
+			line: `${specialForm}`,
+			tokens: [
+				{startIndex: 0, type: `${type}.clj`},
+			],
+		});
+	}
+
+	return testCases;
+}
+
+testTokenization('clojure', [
+	// special forms
+	createTestCases(specialForms, 'keyword'),
+
+	// core symbols
+	createTestCases(coreSymbols, 'keyword'),
+
+	// atoms
+	createTestCases(['false', 'nil', 'true'], 'constant'),
+
+	// keywords
+	createTestCases([':foo', '::bar', ':foo/bar', ':foo.bar/baz'], 'constant'),
+
+	// numbers
+	createTestCases([
+		'42', '+42', '-421',
+		'42N', '+42N', '-42N',
+		'0.42', '+0.42', '-0.42',
+		'42M', '+42M', '-42M',
+		'42.42M', '+42.42M', '-42.42M',
+		'1/42', '+1/42', '-1/42',
+		'0x42af', '+0x42af', '-0x42af',
+		'0x42AF', '+0x42AF', '-0x42AF',
+		'1e2', '1e+2', '1e-2',
+		'+1e2', '+1e+2', '+1e-2',
+		'-1e2', '-1e+2', '-1e-2',
+		'-1.0e2', '-0.1e+2', '-1.01e-2',
+		'1E2', '1E+2', '1E-2',
+		'+1E2', '+1E+2', '+1E-2',
+		'-1E2', '-1E+2', '-1E-2',
+		'-1.0E2', '-0.1E+2', '-1.01E-2',
+		'2r101010', '+2r101010', '-2r101010',
+		'2r101010', '+2r101010', '-2r101010',
+		'8r52', '+8r52', '-8r52',
+		'36rhello', '+36rhello', '-36rhello',
+		'36rz', '+36rz', '-36rz',
+		'36rZ', '+36rZ', '-36rZ',
+	], 'number'),
+
+	// characters
+	createTestCases([
+		'\\1',
+		'\\a',
+		'\\#',
+		'\\\\',
+		'\\\"',
+		'\\(',
+		'\\A',
+		'\\backspace',
+		'\\formfeed',
+		'\\newline',
+		'\\space',
+		'\\return',
+		'\\tab',
+		'\\o123',
+		'\\u1000',
+		'\\uAaAa',
+		'\\u9F9F'
+	], 'string'),
+
+	// strings
+	[
+		{
+			line: '"I\'m a little teapot."',
+			tokens: [
+				{startIndex: 0, type: 'string.clj'},
+			]
+		},
+		{
+			line: '"I\'m a \\"little\\" teapot."',
+			tokens: [
+				{startIndex: 0, type: 'string.clj'},
+				{startIndex: 7, type: 'string.escape.clj'},
+				{startIndex: 9, type: 'string.clj'},
+				{startIndex: 15, type: 'string.escape.clj'},
+				{startIndex: 17, type: 'string.clj'},
+			]
+		}
+	],
+
+	// multi-line strings
+	[
+		{
+			line: '"I\'m',
+			tokens: [
+				{startIndex: 0, type: 'string.clj'},
+			]
+		},
+		{
+			line: '\\"a little\\"',
+			tokens: [
+				{startIndex: 0, type: 'string.escape.clj'},
+				{startIndex: 2, type: 'string.clj'},
+				{startIndex: 10, type: 'string.escape.clj'},
+			]
+		},
+		{
+			line: 'teapot."',
+			tokens: [
+				{startIndex: 0, type: 'string.clj'},
+			]
+		}
+	],
+
+	// strings with other escapes in them (\" \' \\ \b \f \n \r \t)
+	[{
+		line: '"the escape \\" \\\' \\\\ \\b \\f \\n \\r \\t characters"',
+		tokens: [
+			{startIndex: 0, type: 'string.clj'},
+			{startIndex: 12, type: 'string.escape.clj'},
+			{startIndex: 14, type: 'string.clj'},
+			{startIndex: 15, type: 'string.escape.clj'},
+			{startIndex: 17, type: 'string.clj'},
+			{startIndex: 18, type: 'string.escape.clj'},
+			{startIndex: 20, type: 'string.clj'},
+			{startIndex: 21, type: 'string.escape.clj'},
+			{startIndex: 23, type: 'string.clj'},
+			{startIndex: 24, type: 'string.escape.clj'},
+			{startIndex: 26, type: 'string.clj'},
+			{startIndex: 27, type: 'string.escape.clj'},
+			{startIndex: 29, type: 'string.clj'},
+			{startIndex: 30, type: 'string.escape.clj'},
+			{startIndex: 32, type: 'string.clj'},
+			{startIndex: 33, type: 'string.escape.clj'},
+			{startIndex: 35, type: 'string.clj'},
+		]
+	}],
+
+	// comments
+	createTestCases([
+		'; this is an in-line comment.',
+		';; this is a line comment.',
+	], 'comment'),
+
+	// `comment`
+	[
+		{
+			line: '(comment)',
+			tokens: [
+				{startIndex: 0, type: 'comment.clj'},
+			],
+		},
+		{
+			line: 'foo :bar 42',
+			tokens: [
+				{startIndex: 0, type: 'identifier.clj'},
+				{startIndex: 3, type: 'white.clj'},
+				{startIndex: 4, type: 'constant.clj'},
+				{startIndex: 8, type: 'white.clj'},
+				{startIndex: 9, type: 'number.clj'},
+			],
+		},
+		{
+			line: '(comment (foo [bar :baz 1 "qux"]))',
+			tokens: [
+				{startIndex: 0, type: 'comment.clj'},
+			],
+		},
+		{
+			line: '(comments foo bar)',
+			tokens: [
+				{startIndex: 0, type: 'delimiter.parenthesis.clj'},
+				{startIndex: 1, type: 'identifier.clj'},
+				{startIndex: 9, type: 'white.clj'},
+				{startIndex: 10, type: 'identifier.clj'},
+				{startIndex: 13, type: 'white.clj'},
+				{startIndex: 14, type: 'identifier.clj'},
+				{startIndex: 17, type: 'delimiter.parenthesis.clj'},
+			]
+		},
+		{
+			line: '(comment6 foo bar)',
+			tokens: [
+				{startIndex: 0, type: 'delimiter.parenthesis.clj'},
+				{startIndex: 1, type: 'identifier.clj'},
+				{startIndex: 9, type: 'white.clj'},
+				{startIndex: 10, type: 'identifier.clj'},
+				{startIndex: 13, type: 'white.clj'},
+				{startIndex: 14, type: 'identifier.clj'},
+				{startIndex: 17, type: 'delimiter.parenthesis.clj'},
+			]
+		},
+		{
+			line: '(comment foo',
+			tokens: [
+				{startIndex: 0, type: 'comment.clj'},
+			],
+		},
+		{
+			line: 'foo',
+			tokens: [
+				{startIndex: 0, type: 'comment.clj'},
+			],
+		},
+	],
+
+	// reader macro characters
+	createTestCases([
+		'#',
+		'@',
+		'^',
+		'`',
+		'~',
+		"'",
+	], 'meta')
+]);

src/clojure/clojure.ts

@@ -0,0 +1,792 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: ';;',
+	},
+
+	brackets: [
+		['[', ']'],
+		['(', ')'],
+		['{', '}']
+	],
+
+	autoClosingPairs: [
+		{open: '[', close: ']'},
+		{open: '"', close: '"'},
+		{open: '(', close: ')'},
+		{open: '{', close: '}'},
+	],
+
+	surroundingPairs: [
+		{open: '[', close: ']'},
+		{open: '"', close: '"'},
+		{open: '(', close: ')'},
+		{open: '{', close: '}'},
+	],
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.clj',
+
+	brackets: [
+		{open: '[', close: ']', token: 'delimiter.square'},
+		{open: '(', close: ')', token: 'delimiter.parenthesis'},
+		{open: '{', close: '}', token: 'delimiter.curly'},
+	],
+
+	constants: ['true', 'false', 'nil'],
+
+	// delimiters: /[\\\[\]\s"#'(),;@^`{}~]|$/,
+
+	numbers: /^(?:[+\-]?\d+(?:(?:N|(?:[eE][+\-]?\d+))|(?:\.?\d*(?:M|(?:[eE][+\-]?\d+))?)|\/\d+|[xX][0-9a-fA-F]+|r[0-9a-zA-Z]+)?(?=[\\\[\]\s"#'(),;@^`{}~]|$))/,
+
+	characters: /^(?:\\(?:backspace|formfeed|newline|return|space|tab|o[0-7]{3}|u[0-9A-Fa-f]{4}|x[0-9A-Fa-f]{4}|.)?(?=[\\\[\]\s"(),;@^`{}~]|$))/,
+
+	escapes: /^\\(?:["'\\bfnrt]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// simple-namespace := /^[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*/
+	// simple-symbol    := /^(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)/
+	// qualified-symbol := (<simple-namespace>(<.><simple-namespace>)*</>)?<simple-symbol>
+	qualifiedSymbols: /^(?:(?:[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*(?:\.[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)*\/)?(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)*(?=[\\\[\]\s"(),;@^`{}~]|$))/,
+
+	specialForms: [
+		'.',
+		'catch',
+		'def',
+		'do',
+		'if',
+		'monitor-enter',
+		'monitor-exit',
+		'new',
+		'quote',
+		'recur',
+		'set!',
+		'throw',
+		'try',
+		'var',
+	],
+
+	coreSymbols: [
+		'*',
+		'*\'',
+		'*1',
+		'*2',
+		'*3',
+		'*agent*',
+		'*allow-unresolved-vars*',
+		'*assert*',
+		'*clojure-version*',
+		'*command-line-args*',
+		'*compile-files*',
+		'*compile-path*',
+		'*compiler-options*',
+		'*data-readers*',
+		'*default-data-reader-fn*',
+		'*e',
+		'*err*',
+		'*file*',
+		'*flush-on-newline*',
+		'*fn-loader*',
+		'*in*',
+		'*math-context*',
+		'*ns*',
+		'*out*',
+		'*print-dup*',
+		'*print-length*',
+		'*print-level*',
+		'*print-meta*',
+		'*print-namespace-maps*',
+		'*print-readably*',
+		'*read-eval*',
+		'*reader-resolver*',
+		'*source-path*',
+		'*suppress-read*',
+		'*unchecked-math*',
+		'*use-context-classloader*',
+		'*verbose-defrecords*',
+		'*warn-on-reflection*',
+		'+',
+		'+\'',
+		'-',
+		'-\'',
+		'->',
+		'->>',
+		'->ArrayChunk',
+		'->Eduction',
+		'->Vec',
+		'->VecNode',
+		'->VecSeq',
+		'-cache-protocol-fn',
+		'-reset-methods',
+		'..',
+		'/',
+		'<',
+		'<=',
+		'=',
+		'==',
+		'>',
+		'>=',
+		'EMPTY-NODE',
+		'Inst',
+		'StackTraceElement->vec',
+		'Throwable->map',
+		'accessor',
+		'aclone',
+		'add-classpath',
+		'add-watch',
+		'agent',
+		'agent-error',
+		'agent-errors',
+		'aget',
+		'alength',
+		'alias',
+		'all-ns',
+		'alter',
+		'alter-meta!',
+		'alter-var-root',
+		'amap',
+		'ancestors',
+		'and',
+		'any?',
+		'apply',
+		'areduce',
+		'array-map',
+		'as->',
+		'aset',
+		'aset-boolean',
+		'aset-byte',
+		'aset-char',
+		'aset-double',
+		'aset-float',
+		'aset-int',
+		'aset-long',
+		'aset-short',
+		'assert',
+		'assoc',
+		'assoc!',
+		'assoc-in',
+		'associative?',
+		'atom',
+		'await',
+		'await-for',
+		'await1',
+		'bases',
+		'bean',
+		'bigdec',
+		'bigint',
+		'biginteger',
+		'binding',
+		'bit-and',
+		'bit-and-not',
+		'bit-clear',
+		'bit-flip',
+		'bit-not',
+		'bit-or',
+		'bit-set',
+		'bit-shift-left',
+		'bit-shift-right',
+		'bit-test',
+		'bit-xor',
+		'boolean',
+		'boolean-array',
+		'boolean?',
+		'booleans',
+		'bound-fn',
+		'bound-fn*',
+		'bound?',
+		'bounded-count',
+		'butlast',
+		'byte',
+		'byte-array',
+		'bytes',
+		'bytes?',
+		'case',
+		'cast',
+		'cat',
+		'char',
+		'char-array',
+		'char-escape-string',
+		'char-name-string',
+		'char?',
+		'chars',
+		'chunk',
+		'chunk-append',
+		'chunk-buffer',
+		'chunk-cons',
+		'chunk-first',
+		'chunk-next',
+		'chunk-rest',
+		'chunked-seq?',
+		'class',
+		'class?',
+		'clear-agent-errors',
+		'clojure-version',
+		'coll?',
+		'comment',
+		'commute',
+		'comp',
+		'comparator',
+		'compare',
+		'compare-and-set!',
+		'compile',
+		'complement',
+		'completing',
+		'concat',
+		'cond',
+		'cond->',
+		'cond->>',
+		'condp',
+		'conj',
+		'conj!',
+		'cons',
+		'constantly',
+		'construct-proxy',
+		'contains?',
+		'count',
+		'counted?',
+		'create-ns',
+		'create-struct',
+		'cycle',
+		'dec',
+		'dec\'',
+		'decimal?',
+		'declare',
+		'dedupe',
+		'default-data-readers',
+		'definline',
+		'definterface',
+		'defmacro',
+		'defmethod',
+		'defmulti',
+		'defn',
+		'defn-',
+		'defonce',
+		'defprotocol',
+		'defrecord',
+		'defstruct',
+		'deftype',
+		'delay',
+		'delay?',
+		'deliver',
+		'denominator',
+		'deref',
+		'derive',
+		'descendants',
+		'destructure',
+		'disj',
+		'disj!',
+		'dissoc',
+		'dissoc!',
+		'distinct',
+		'distinct?',
+		'doall',
+		'dorun',
+		'doseq',
+		'dosync',
+		'dotimes',
+		'doto',
+		'double',
+		'double-array',
+		'double?',
+		'doubles',
+		'drop',
+		'drop-last',
+		'drop-while',
+		'eduction',
+		'empty',
+		'empty?',
+		'ensure',
+		'ensure-reduced',
+		'enumeration-seq',
+		'error-handler',
+		'error-mode',
+		'eval',
+		'even?',
+		'every-pred',
+		'every?',
+		'ex-data',
+		'ex-info',
+		'extend',
+		'extend-protocol',
+		'extend-type',
+		'extenders',
+		'extends?',
+		'false?',
+		'ffirst',
+		'file-seq',
+		'filter',
+		'filterv',
+		'find',
+		'find-keyword',
+		'find-ns',
+		'find-protocol-impl',
+		'find-protocol-method',
+		'find-var',
+		'first',
+		'flatten',
+		'float',
+		'float-array',
+		'float?',
+		'floats',
+		'flush',
+		'fn',
+		'fn?',
+		'fnext',
+		'fnil',
+		'for',
+		'force',
+		'format',
+		'frequencies',
+		'future',
+		'future-call',
+		'future-cancel',
+		'future-cancelled?',
+		'future-done?',
+		'future?',
+		'gen-class',
+		'gen-interface',
+		'gensym',
+		'get',
+		'get-in',
+		'get-method',
+		'get-proxy-class',
+		'get-thread-bindings',
+		'get-validator',
+		'group-by',
+		'halt-when',
+		'hash',
+		'hash-combine',
+		'hash-map',
+		'hash-ordered-coll',
+		'hash-set',
+		'hash-unordered-coll',
+		'ident?',
+		'identical?',
+		'identity',
+		'if-let',
+		'if-not',
+		'if-some',
+		'ifn?',
+		'import',
+		'in-ns',
+		'inc',
+		'inc\'',
+		'indexed?',
+		'init-proxy',
+		'inst-ms',
+		'inst-ms*',
+		'inst?',
+		'instance?',
+		'int',
+		'int-array',
+		'int?',
+		'integer?',
+		'interleave',
+		'intern',
+		'interpose',
+		'into',
+		'into-array',
+		'ints',
+		'io!',
+		'isa?',
+		'iterate',
+		'iterator-seq',
+		'juxt',
+		'keep',
+		'keep-indexed',
+		'key',
+		'keys',
+		'keyword',
+		'keyword?',
+		'last',
+		'lazy-cat',
+		'lazy-seq',
+		'let',
+		'letfn',
+		'line-seq',
+		'list',
+		'list*',
+		'list?',
+		'load',
+		'load-file',
+		'load-reader',
+		'load-string',
+		'loaded-libs',
+		'locking',
+		'long',
+		'long-array',
+		'longs',
+		'loop',
+		'macroexpand',
+		'macroexpand-1',
+		'make-array',
+		'make-hierarchy',
+		'map',
+		'map-entry?',
+		'map-indexed',
+		'map?',
+		'mapcat',
+		'mapv',
+		'max',
+		'max-key',
+		'memfn',
+		'memoize',
+		'merge',
+		'merge-with',
+		'meta',
+		'method-sig',
+		'methods',
+		'min',
+		'min-key',
+		'mix-collection-hash',
+		'mod',
+		'munge',
+		'name',
+		'namespace',
+		'namespace-munge',
+		'nat-int?',
+		'neg-int?',
+		'neg?',
+		'newline',
+		'next',
+		'nfirst',
+		'nil?',
+		'nnext',
+		'not',
+		'not-any?',
+		'not-empty',
+		'not-every?',
+		'not=',
+		'ns',
+		'ns-aliases',
+		'ns-imports',
+		'ns-interns',
+		'ns-map',
+		'ns-name',
+		'ns-publics',
+		'ns-refers',
+		'ns-resolve',
+		'ns-unalias',
+		'ns-unmap',
+		'nth',
+		'nthnext',
+		'nthrest',
+		'num',
+		'number?',
+		'numerator',
+		'object-array',
+		'odd?',
+		'or',
+		'parents',
+		'partial',
+		'partition',
+		'partition-all',
+		'partition-by',
+		'pcalls',
+		'peek',
+		'persistent!',
+		'pmap',
+		'pop',
+		'pop!',
+		'pop-thread-bindings',
+		'pos-int?',
+		'pos?',
+		'pr',
+		'pr-str',
+		'prefer-method',
+		'prefers',
+		'primitives-classnames',
+		'print',
+		'print-ctor',
+		'print-dup',
+		'print-method',
+		'print-simple',
+		'print-str',
+		'printf',
+		'println',
+		'println-str',
+		'prn',
+		'prn-str',
+		'promise',
+		'proxy',
+		'proxy-call-with-super',
+		'proxy-mappings',
+		'proxy-name',
+		'proxy-super',
+		'push-thread-bindings',
+		'pvalues',
+		'qualified-ident?',
+		'qualified-keyword?',
+		'qualified-symbol?',
+		'quot',
+		'rand',
+		'rand-int',
+		'rand-nth',
+		'random-sample',
+		'range',
+		'ratio?',
+		'rational?',
+		'rationalize',
+		're-find',
+		're-groups',
+		're-matcher',
+		're-matches',
+		're-pattern',
+		're-seq',
+		'read',
+		'read-line',
+		'read-string',
+		'reader-conditional',
+		'reader-conditional?',
+		'realized?',
+		'record?',
+		'reduce',
+		'reduce-kv',
+		'reduced',
+		'reduced?',
+		'reductions',
+		'ref',
+		'ref-history-count',
+		'ref-max-history',
+		'ref-min-history',
+		'ref-set',
+		'refer',
+		'refer-clojure',
+		'reify',
+		'release-pending-sends',
+		'rem',
+		'remove',
+		'remove-all-methods',
+		'remove-method',
+		'remove-ns',
+		'remove-watch',
+		'repeat',
+		'repeatedly',
+		'replace',
+		'replicate',
+		'require',
+		'reset!',
+		'reset-meta!',
+		'reset-vals!',
+		'resolve',
+		'rest',
+		'restart-agent',
+		'resultset-seq',
+		'reverse',
+		'reversible?',
+		'rseq',
+		'rsubseq',
+		'run!',
+		'satisfies?',
+		'second',
+		'select-keys',
+		'send',
+		'send-off',
+		'send-via',
+		'seq',
+		'seq?',
+		'seqable?',
+		'seque',
+		'sequence',
+		'sequential?',
+		'set',
+		'set-agent-send-executor!',
+		'set-agent-send-off-executor!',
+		'set-error-handler!',
+		'set-error-mode!',
+		'set-validator!',
+		'set?',
+		'short',
+		'short-array',
+		'shorts',
+		'shuffle',
+		'shutdown-agents',
+		'simple-ident?',
+		'simple-keyword?',
+		'simple-symbol?',
+		'slurp',
+		'some',
+		'some->',
+		'some->>',
+		'some-fn',
+		'some?',
+		'sort',
+		'sort-by',
+		'sorted-map',
+		'sorted-map-by',
+		'sorted-set',
+		'sorted-set-by',
+		'sorted?',
+		'special-symbol?',
+		'spit',
+		'split-at',
+		'split-with',
+		'str',
+		'string?',
+		'struct',
+		'struct-map',
+		'subs',
+		'subseq',
+		'subvec',
+		'supers',
+		'swap!',
+		'swap-vals!',
+		'symbol',
+		'symbol?',
+		'sync',
+		'tagged-literal',
+		'tagged-literal?',
+		'take',
+		'take-last',
+		'take-nth',
+		'take-while',
+		'test',
+		'the-ns',
+		'thread-bound?',
+		'time',
+		'to-array',
+		'to-array-2d',
+		'trampoline',
+		'transduce',
+		'transient',
+		'tree-seq',
+		'true?',
+		'type',
+		'unchecked-add',
+		'unchecked-add-int',
+		'unchecked-byte',
+		'unchecked-char',
+		'unchecked-dec',
+		'unchecked-dec-int',
+		'unchecked-divide-int',
+		'unchecked-double',
+		'unchecked-float',
+		'unchecked-inc',
+		'unchecked-inc-int',
+		'unchecked-int',
+		'unchecked-long',
+		'unchecked-multiply',
+		'unchecked-multiply-int',
+		'unchecked-negate',
+		'unchecked-negate-int',
+		'unchecked-remainder-int',
+		'unchecked-short',
+		'unchecked-subtract',
+		'unchecked-subtract-int',
+		'underive',
+		'unquote',
+		'unquote-splicing',
+		'unreduced',
+		'unsigned-bit-shift-right',
+		'update',
+		'update-in',
+		'update-proxy',
+		'uri?',
+		'use',
+		'uuid?',
+		'val',
+		'vals',
+		'var-get',
+		'var-set',
+		'var?',
+		'vary-meta',
+		'vec',
+		'vector',
+		'vector-of',
+		'vector?',
+		'volatile!',
+		'volatile?',
+		'vreset!',
+		'vswap!',
+		'when',
+		'when-first',
+		'when-let',
+		'when-not',
+		'when-some',
+		'while',
+		'with-bindings',
+		'with-bindings*',
+		'with-in-str',
+		'with-loading-context',
+		'with-local-vars',
+		'with-meta',
+		'with-open',
+		'with-out-str',
+		'with-precision',
+		'with-redefs',
+		'with-redefs-fn',
+		'xml-seq',
+		'zero?',
+		'zipmap',
+	],
+
+	tokenizer: {
+		root: [
+			// whitespaces and comments
+			{include: '@whitespace'},
+
+			// numbers
+			[/@numbers/, 'number'],
+
+			// characters
+			[/@characters/, 'string'],
+
+			// strings
+			{include: '@string'},
+
+			// brackets
+			[/[()\[\]{}]/, '@brackets'],
+
+			// regular expressions
+			[/\/#"(?:\.|(?:")|[^"\n])*"\/g/, 'regexp'],
+
+			// reader macro characters
+			[/[#'@^`~]/, 'meta'],
+
+			// symbols
+			[/@qualifiedSymbols/, {
+					cases: {
+						'^:.+$': 'constant',  // Clojure keywords (e.g., `:foo/bar`)
+						'@specialForms': 'keyword',
+						'@coreSymbols': 'keyword',
+						'@constants': 'constant',
+						'@default': 'identifier',
+					},
+				},
+			],
+		],
+
+		whitespace: [
+			[/\s+/, 'white'],
+			[/;.*$/, 'comment'],
+			[/\(comment\b/, 'comment', '@comment'],
+		],
+
+		comment: [
+			[/\(/, 'comment', '@push'],
+			[/\)/, 'comment', '@pop'],
+			[/[^()]/, 'comment'],
+		],
+
+		string: [
+			[/"/, 'string', '@multiLineString'],
+		],
+
+		multiLineString: [
+			[/"/, 'string', '@popall'],
+			[/@escapes/, 'string.escape'],
+			[/./, 'string']
+		],
+	},
+};

src/coffee/coffee.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'coffeescript',
 	extensions: ['.coffee'],
 	aliases: ['CoffeeScript', 'coffeescript', 'coffee'],
 	mimetypes: ['text/x-coffeescript', 'text/coffeescript'],
-	loader: () => _monaco.Promise.wrap(import('./coffee'))
+	loader: () => import('./coffee')
 });

src/cpp/cpp.contribution.ts

@@ -6,18 +6,15 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'c',
 	extensions: ['.c', '.h'],
 	aliases: ['C', 'c'],
-	loader: () => _monaco.Promise.wrap(import('./cpp'))
+	loader: () => import('./cpp')
 });
 registerLanguage({
 	id: 'cpp',
 	extensions: ['.cpp', '.cc', '.cxx', '.hpp', '.hh', '.hxx'],
 	aliases: ['C++', 'Cpp', 'cpp'],
-	loader: () => _monaco.Promise.wrap(import('./cpp'))
+	loader: () => import('./cpp')
 });

src/cpp/cpp.test.ts

@@ -391,17 +391,19 @@ testTokenization('cpp', [
 	[{
 		line: '#include<iostream>',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.cpp' },
-			{ startIndex: 8, type: 'delimiter.angle.cpp' },
-			{ startIndex: 9, type: 'identifier.cpp' },
-			{ startIndex: 17, type: 'delimiter.angle.cpp' }
+			{ startIndex: 0, type: 'keyword.directive.include.cpp' },
+			{ startIndex: 8, type: 'keyword.directive.include.begin.cpp' },
+			{ startIndex: 9, type: 'string.include.identifier.cpp' },
+			{ startIndex: 17, type: 'keyword.directive.include.end.cpp' }
 		]
 	}, {
 		line: '#include "/path/to/my/file.h"',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.cpp' },
+			{ startIndex: 0, type: 'keyword.directive.include.cpp' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 9, type: 'string.cpp' }
+			{ startIndex: 9, type: 'keyword.directive.include.begin.cpp' },
+			{ startIndex: 10, type: 'string.include.identifier.cpp' },
+			{ startIndex: 28, type: 'keyword.directive.include.end.cpp' }
 		]
 	}, {
 		line: '',

src/cpp/cpp.ts

@@ -268,6 +268,8 @@ export const language = <ILanguage>{
 			// [[ attributes ]].
 			[/\[\[.*\]\]/, 'annotation'],
 
+			[/^\s*#include/, { token: 'keyword.directive.include', next: '@include' }],
+
 			// Preprocessor directive
 			[/^\s*#\s*\w+/, 'keyword'],
 
@@ -338,6 +340,11 @@ export const language = <ILanguage>{
 				}
 			],
 			[/.*/, 'string.raw']
+		],
+
+		include: [
+			[/(\s*)(<)([^<>]*)(>)/, ['', 'keyword.directive.include.begin', 'string.include.identifier', { token: 'keyword.directive.include.end', next: '@pop'}]],
+			[/(\s*)(")([^"]*)(")/, ['', 'keyword.directive.include.begin', 'string.include.identifier', { token: 'keyword.directive.include.end', next: '@pop'}]]
 		]
 	},
 };

src/csharp/csharp.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'csharp',
-	extensions: ['.cs', '.csx'],
+	extensions: ['.cs', '.csx', '.cake'],
 	aliases: ['C#', 'csharp'],
-	loader: () => _monaco.Promise.wrap(import('./csharp'))
+	loader: () => import('./csharp')
 });

src/csp/csp.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'csp',
 	extensions: [],
 	aliases: ['CSP', 'csp'],
-	loader: () => _monaco.Promise.wrap(import('./csp'))
+	loader: () => import('./csp')
 });

src/css/css.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'css',
 	extensions: ['.css'],
 	aliases: ['CSS', 'css'],
 	mimetypes: ['text/css'],
-	loader: () => _monaco.Promise.wrap(import('./css'))
+	loader: () => import('./css')
 });

src/dockerfile/dockerfile.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'dockerfile',
 	extensions: ['.dockerfile'],
 	filenames: ['Dockerfile'],
 	aliases: ['Dockerfile'],
-	loader: () => _monaco.Promise.wrap(import('./dockerfile'))
+	loader: () => import('./dockerfile')
 });

src/dockerfile/dockerfile.ts

@@ -34,12 +34,6 @@ export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.dockerfile',
 
-	instructions: /FROM|MAINTAINER|RUN|EXPOSE|ENV|ADD|ARG|VOLUME|LABEL|USER|WORKDIR|COPY|CMD|STOPSIGNAL|SHELL|HEALTHCHECK|ENTRYPOINT/,
-
-	instructionAfter: /ONBUILD/,
-
-	variableAfter: /ENV/,
-
 	variable: /\${?[\w]+}?/,
 
 	tokenizer: {
@@ -47,13 +41,9 @@ export const language = <ILanguage>{
 			{ include: '@whitespace' },
 			{ include: '@comment' },
 
-			[/(@instructionAfter)(\s+)/, ['keyword', { token: '', next: '@instructions' }]],
-			['', 'keyword', '@instructions']
-		],
-
-		instructions: [
-			[/(@variableAfter)(\s+)([\w]+)/, ['keyword', '', { token: 'variable', next: '@arguments' }]],
-			[/(@instructions)/, 'keyword', '@arguments']
+			[/(ONBUILD)(\s+)/, ['keyword', '']],
+			[/(ENV)(\s+)([\w]+)/, ['keyword', '', { token: 'variable', next: '@arguments' }]],
+			[/(FROM|MAINTAINER|RUN|EXPOSE|ENV|ADD|ARG|VOLUME|LABEL|USER|WORKDIR|COPY|CMD|STOPSIGNAL|SHELL|HEALTHCHECK|ENTRYPOINT)/, { token: 'keyword', next: '@arguments' }]
 		],
 
 		arguments: [

src/fsharp/fsharp.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'fsharp',
 	extensions: ['.fs', '.fsi', '.ml', '.mli', '.fsx', '.fsscript'],
 	aliases: ['F#', 'FSharp', 'fsharp'],
-	loader: () => _monaco.Promise.wrap(import('./fsharp'))
+	loader: () => import('./fsharp')
 });

src/fsharp/fsharp.test.ts

@@ -442,5 +442,20 @@ testTokenization('fsharp', [
 		tokens: [
 			{ startIndex: 0, type: 'number.float.fs' }
 		]
+	}],
+
+	[{
+		line: '(* This operator -> (*) should be ignored *) let i = 0',
+		tokens: [
+			{ startIndex: 0, type: 'comment.fs' },
+			{ startIndex: 44, type: '' },
+			{ startIndex: 45, type: 'keyword.let.fs' },
+			{ startIndex: 48, type: '' },
+			{ startIndex: 49, type: 'identifier.fs' },
+			{ startIndex: 50, type: '' },
+			{ startIndex: 51, type: 'delimiter.fs' },
+			{ startIndex: 52, type: '' },
+			{ startIndex: 53, type: 'number.fs' }
+		]
 	}]
 ]);

src/fsharp/fsharp.ts

@@ -132,9 +132,11 @@ export const language = <ILanguage>{
 		],
 
 		comment: [
-			[/[^\*]+/, 'comment'],
+			[/[^*(]+/, 'comment'],
 			[/\*\)/, 'comment', '@pop'],
-			[/\*/, 'comment']
+			[/\*/, 'comment'],
+			[/\(\*\)/, 'comment'],
+			[/\(/, 'comment']
 		],
 
 		string: [

src/go/go.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'go',
 	extensions: ['.go'],
 	aliases: ['Go'],
-	loader: () => _monaco.Promise.wrap(import('./go'))
+	loader: () => import('./go')
 });

src/graphql/graphql.contribution.ts

@@ -0,0 +1,15 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'graphql',
+	extensions: ['.graphql', '.gql'],
+	aliases: ['GraphQL', 'graphql', 'gql'],
+	mimetypes: ['application/graphql'],
+	loader: () => import('./graphql')
+});

src/graphql/graphql.test.ts

@@ -0,0 +1,143 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('graphql', [
+	// Keywords
+	[{
+		line: 'scalar Date',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.gql' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'type.identifier.gql' },
+		]
+	}],
+
+	// Root schema definition
+	[{
+		line: 'schema { query: Query, mutation: Mutation subscription: Subscription }',
+		tokens: [
+			{ startIndex: 0, type: "keyword.gql" },
+			{ startIndex: 6, type: "" },
+			{ startIndex: 7, type: "delimiter.curly.gql" },
+			{ startIndex: 8, type: "" },
+			{ startIndex: 9, type: "keyword.gql" }, // this should be identifier
+			{ startIndex: 14, type: "operator.gql" },
+			{ startIndex: 15, type: "" },
+			{ startIndex: 16, type: "type.identifier.gql" },
+			{ startIndex: 21, type: "delimiter.gql" },
+			{ startIndex: 22, type: "" },
+			{ startIndex: 23, type: "keyword.gql" }, // this should be identifier
+			{ startIndex: 31, type: "operator.gql" },
+			{ startIndex: 32, type: "" },
+			{ startIndex: 33, type: "type.identifier.gql" },
+			{ startIndex: 41, type: "" },
+			{ startIndex: 42, type: "keyword.gql" }, // this should be identifier
+			{ startIndex: 54, type: "operator.gql" },
+			{ startIndex: 55, type: "" },
+			{ startIndex: 56, type: "type.identifier.gql" },
+			{ startIndex: 68, type: "" },
+			{ startIndex: 69, type: "delimiter.curly.gql" },
+		]
+	}],
+
+	[{
+		line: `query testQuery($intValue:Int=3){value(arg:{string:"string" int:$intValue}){field1 field2}}`,
+		tokens: [
+			{ startIndex: 0, type: "keyword.gql" },                 // 'query'
+			{ startIndex: 5, type: "" },                            // ' '
+			{ startIndex: 6, type: "identifier.gql" },              // 'testQuery'
+			{ startIndex: 15, type: "delimiter.parenthesis.gql" },  // '('
+			{ startIndex: 16, type: "identifier.gql" },             // '$intValue'
+			{ startIndex: 25, type: "operator.gql" },               // ':'
+			{ startIndex: 26, type: "keyword.gql" },                // 'Int'
+			{ startIndex: 29, type: "operator.gql" },               // '='
+			{ startIndex: 30, type: "number.gql" },                 // '3'
+			{ startIndex: 31, type: "delimiter.parenthesis.gql" },  // ')'
+			{ startIndex: 32, type: "delimiter.curly.gql" },        // '{'
+			{ startIndex: 33, type: "identifier.gql" },             // 'value'
+			{ startIndex: 38, type: "delimiter.parenthesis.gql" },  // '('
+			{ startIndex: 39, type: "identifier.gql" },             // 'arg'
+			{ startIndex: 42, type: "operator.gql" },               // ':'
+			{ startIndex: 43, type: "delimiter.curly.gql" },        // '{'
+			{ startIndex: 44, type: "identifier.gql" },             // 'string'
+			{ startIndex: 50, type: "operator.gql" },               // ':'
+			{ startIndex: 51, type: "string.quote.gql" },           // '"'
+			{ startIndex: 52, type: "string.gql" },                 // 'string'
+			{ startIndex: 58, type: "string.quote.gql" },           // '"'
+			{ startIndex: 59, type: "" },                           // ' '
+			{ startIndex: 60, type: "identifier.gql" },             // 'int'
+			{ startIndex: 63, type: "operator.gql" },               // ':'
+			{ startIndex: 64, type: "identifier.gql" },             // '$intValue'
+			{ startIndex: 73, type: "delimiter.curly.gql" },        // '}'
+			{ startIndex: 74, type: "delimiter.parenthesis.gql" },  // ')'
+			{ startIndex: 75, type: "delimiter.curly.gql" },        // '{'
+			{ startIndex: 76, type: "identifier.gql" },             // 'field1'
+			{ startIndex: 82, type: "" },                           // ' '
+			{ startIndex: 83, type: "identifier.gql" },             // 'field2'
+			{ startIndex: 89, type: "delimiter.curly.gql" },        // '}}'
+		],
+	}],
+
+	// More complex test:
+	//   """
+	//   Node interface
+	//   - allows (re)fetch arbitrary entity only by ID
+	//   """
+	//   interface Node {
+	//     id: ID!
+	//   }
+	[
+		{
+			line: '"""',
+			tokens: [
+				{ startIndex: 0, type: "string.gql" }
+			],
+		},
+		{
+			line: 'This is MarkDown',
+			tokens: [
+				{ startIndex: 0, type: "" }
+			],
+		},
+		{
+			line: '"""',
+			tokens: [
+				{ startIndex: 0, type: "string.gql" }
+			],
+		},
+		{
+			line: 'interface Node {',
+			tokens: [
+				{ startIndex: 0, type: "keyword.gql" },
+				{ startIndex: 9, type: "" },
+				{ startIndex: 10, type: "type.identifier.gql" },
+				{ startIndex: 14, type: "" },
+				{ startIndex: 15, type: "delimiter.curly.gql" },
+			],
+		},
+		{
+			line: '  id: ID!',
+			tokens: [
+				{ startIndex: 0, type: "" },
+				{ startIndex: 2, type: "identifier.gql" },
+				{ startIndex: 4, type: "operator.gql" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "keyword.gql" },
+				{ startIndex: 8, type: "operator.gql" },
+			],
+		},
+		{
+			line: '}',
+			tokens: [
+				{ startIndex: 0, type: "delimiter.curly.gql", },
+			],
+		},
+	]
+
+]);

src/graphql/graphql.ts

@@ -0,0 +1,141 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"""', close: '"""', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"""', close: '"""' },
+		{ open: '"', close: '"' },
+	],
+	folding: {
+		offSide: true
+	}
+};
+
+export const language = <ILanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.gql',
+
+	keywords: [
+		'null', 'true', 'false',
+		'query', 'mutation', 'subscription',
+		'extend', 'schema', 'directive',
+		'scalar', 'type', 'interface', 'union', 'enum', 'input', 'implements',
+		'fragment', 'on',
+	],
+
+	typeKeywords: ['Int', 'Float', 'String', 'Boolean', 'ID'],
+
+	directiveLocations: [
+		'SCHEMA', 'SCALAR', 'OBJECT', 'FIELD_DEFINITION', 'ARGUMENT_DEFINITION',
+		'INTERFACE', 'UNION', 'ENUM', 'ENUM_VALUE', 'INPUT_OBJECT', 'INPUT_FIELD_DEFINITION',
+		'QUERY', 'MUTATION', 'SUBSCRIPTION', 'FIELD', 'FRAGMENT_DEFINITION',
+		'FRAGMENT_SPREAD', 'INLINE_FRAGMENT', 'VARIABLE_DEFINITION',
+	],
+
+	operators: ['=', '!', '?', ':', '&', '|'],
+
+	// we include these common regular expressions
+	symbols: /[=!?:&|]+/,
+
+	// https://facebook.github.io/graphql/draft/#sec-String-Value
+	escapes: /\\(?:["\\\/bfnrt]|u[0-9A-Fa-f]{4})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/[a-z_$][\w$]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@default': 'identifier',
+					},
+				},
+			],
+			[
+				/[A-Z][\w\$]*/,
+				{
+					cases: {
+						'@typeKeywords': 'keyword',
+						'@default': 'type.identifier',
+					},
+				},
+			], // to show class names nicely
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[
+				/@symbols/,
+				{ cases: { '@operators': 'operator', '@default': '' } },
+			],
+
+			// @ annotations.
+			// As an example, we emit a debugging log message on these tokens.
+			// Note: message are supressed during the first load -- change some lines to see them.
+			[
+				/@\s*[a-zA-Z_\$][\w\$]*/,
+				{ token: 'annotation', log: 'annotation token: $0' },
+			],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			[/"""/,
+				{ token: 'string', next: '@mlstring', nextEmbedded: 'markdown' }
+			],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, { token: 'string.quote', bracket: '@open', next: '@string' }],
+		],
+
+		mlstring: [
+			[/[^"]+/, 'string'],
+			['"""', { token: 'string', next: '@pop', nextEmbedded: '@pop' }]
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, { token: 'string.quote', bracket: '@close', next: '@pop' }],
+		],
+
+		whitespace: [[/[ \t\r\n]+/, ''], [/#.*$/, 'comment']],
+	},
+};

src/handlebars/handlebars.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'handlebars',
 	extensions: ['.handlebars', '.hbs'],
 	aliases: ['Handlebars', 'handlebars'],
 	mimetypes: ['text/x-handlebars-template'],
-	loader: () => _monaco.Promise.wrap(import('./handlebars'))
+	loader: () => import('./handlebars')
 });

src/html/html.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'html',
 	extensions: ['.html', '.htm', '.shtml', '.xhtml', '.mdoc', '.jsp', '.asp', '.aspx', '.jshtm'],
 	aliases: ['HTML', 'htm', 'html', 'xhtml'],
 	mimetypes: ['text/html', 'text/x-jshtm', 'text/template', 'text/ng-template'],
-	loader: () => _monaco.Promise.wrap(import('./html'))
+	loader: () => import('./html')
 });

src/html/html.test.ts

@@ -7,7 +7,7 @@
 
 import { testTokenization } from '../test/testRunner';
 
-testTokenization(['html', 'css'], [
+testTokenization(['html', 'css', 'javascript'], [
 
 	// Open Start Tag #1'
 	[{
@@ -147,7 +147,13 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: 'attribute.value.html' },
 			{ startIndex: 30, type: 'delimiter.html' },
-			{ startIndex: 31, type: '' },
+			{ startIndex: 31, type: 'keyword.js' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'identifier.js' },
+			{ startIndex: 36, type: 'delimiter.js' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'number.js' },
+			{ startIndex: 40, type: 'delimiter.js' },
 			{ startIndex: 41, type: 'delimiter.html' },
 			{ startIndex: 43, type: 'tag.html' },
 			{ startIndex: 49, type: 'delimiter.html' }
@@ -169,7 +175,13 @@ testTokenization(['html', 'css'], [
 	}, {
 		line: 'var i= 10;',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
 		]
 	}, {
 		line: '</script>',
@@ -191,7 +203,14 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: 'attribute.value.html' },
 			{ startIndex: 30, type: 'delimiter.html' },
-			{ startIndex: 31, type: '' },
+			{ startIndex: 31, type: 'keyword.js' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'identifier.js' },
+			{ startIndex: 36, type: 'delimiter.js' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'number.js' },
+			{ startIndex: 40, type: 'delimiter.js' },
+
 		]
 	}, {
 		line: '</script>',
@@ -217,7 +236,13 @@ testTokenization(['html', 'css'], [
 	}, {
 		line: 'var i= 10;</script>',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
 			{ startIndex: 10, type: 'delimiter.html' },
 			{ startIndex: 12, type: 'tag.html' },
 			{ startIndex: 18, type: 'delimiter.html' }
@@ -254,14 +279,14 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
 			{ startIndex: 9, type: 'delimiter.html' },
 			{ startIndex: 11, type: 'tag.html' },
 			{ startIndex: 17, type: 'delimiter.html' },
 			// { startIndex:18, type: 'delimiter.html' },
 			{ startIndex: 19, type: 'tag.html' },
 			{ startIndex: 25, type: 'delimiter.html' },
-			{ startIndex: 26, type: '' },
+			{ startIndex: 26, type: 'identifier.js' },
 			{ startIndex: 27, type: 'delimiter.html' },
 			{ startIndex: 29, type: 'tag.html' },
 			{ startIndex: 35, type: 'delimiter.html' }
@@ -292,7 +317,13 @@ testTokenization(['html', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.js' },
+			{ startIndex: 13, type: 'delimiter.js' },
+			{ startIndex: 14, type: '' },
+			{ startIndex: 15, type: 'number.js' },
+			{ startIndex: 17, type: 'delimiter.js' },
 			{ startIndex: 18, type: 'delimiter.html' },
 			{ startIndex: 20, type: 'tag.html' },
 			{ startIndex: 26, type: 'delimiter.html' }

src/ini/ini.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'ini',
 	extensions: ['.ini', '.properties', '.gitconfig'],
 	filenames: ['config', '.gitattributes', '.gitconfig', '.editorconfig'],
 	aliases: ['Ini', 'ini'],
-	loader: () => _monaco.Promise.wrap(import('./ini'))
+	loader: () => import('./ini')
 });

src/java/java.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'java',
 	extensions: ['.java', '.jav'],
 	aliases: ['Java', 'java'],
 	mimetypes: ['text/x-java-source', 'text/x-java'],
-	loader: () => _monaco.Promise.wrap(import('./java'))
+	loader: () => import('./java')
 });

src/javascript/javascript.contribution.ts

@@ -0,0 +1,17 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'javascript',
+	extensions: ['.js', '.es6', '.jsx'],
+	firstLine: '^#!.*\\bnode',
+	filenames: ['jakefile'],
+	aliases: ['JavaScript', 'javascript', 'js'],
+	mimetypes: ['text/javascript'],
+	loader: () => import('./javascript')
+});

src/javascript/javascript.test.ts

@@ -0,0 +1,662 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('javascript', [
+	// Keywords
+	[{
+		line: 'var x = function() { };',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 16, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 18, type: '' },
+			{ startIndex: 19, type: 'delimiter.bracket.js' },
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'delimiter.bracket.js' },
+			{ startIndex: 22, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '    var    ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'keyword.js' },
+			{ startIndex: 7, type: '' }
+		]
+	}],
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '// a comment /*',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '// a comment /**',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: 'var x = 1; // my comment // is a nice one',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'comment.js' }
+		]
+	}],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.js' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.js' },
+			{ startIndex: 32, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.js' },
+			{ startIndex: 8, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'x = /*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.js' }
+		]
+	}],
+
+	// Comments - range comment, multi lines
+	[{
+		line: '/* a multiline comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}, {
+		line: 'can actually span',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}, {
+		line: 'multiple lines */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* start a comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.js' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}, {
+		line: 'and end it */ var a = 2;',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.js' },
+			{ startIndex: 17, type: '' },
+			{ startIndex: 18, type: 'identifier.js' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'delimiter.js' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.js' },
+			{ startIndex: 23, type: 'delimiter.js' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: 'var a = \'a\';',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.js' },
+			{ startIndex: 11, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '"use strict";',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 12, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'b = a + " \'cool\'  "',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.js' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 10, type: 'string.escape.js' },
+			{ startIndex: 12, type: 'string.js' },
+			{ startIndex: 18, type: 'string.escape.js' },
+			{ startIndex: 20, type: 'string.js' },
+		]
+	}],
+
+	[{
+		line: '\'\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 2, type: 'string.invalid.js' },
+		]
+	}],
+
+	[{
+		line: '\'\\\'\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 1, type: 'string.escape.js' },
+			{ startIndex: 3, type: 'string.js' },
+		]
+	}],
+
+	[{
+		line: '\'be careful \\not to escape\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 12, type: 'string.escape.js' },
+			{ startIndex: 14, type: 'string.js' },
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: ' 0',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: ' 0 ',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 1, type: 'number.js' },
+			{ startIndex: 2, type: '' }
+		]
+	}],
+
+	[{
+		line: '0 ',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: 'delimiter.js' },
+			{ startIndex: 2, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 3, type: 'delimiter.js' },
+			{ startIndex: 4, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '0123',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.js' }
+		]
+	}],
+
+	[{
+		line: '01239',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.js' },
+			{ startIndex: 4, type: 'number.js' }
+		]
+	}],
+
+	[{
+		line: '0o123',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.js' }
+		]
+	}],
+
+	[{
+		line: '0O123',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.js' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: 'identifier.js' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.js' }
+		]
+	}],
+
+	[{
+		line: '0X123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.js' }
+		]
+	}],
+
+	[{
+		line: '0b101',
+		tokens: [
+			{ startIndex: 0, type: 'number.binary.js' }
+		]
+	}],
+
+	[{
+		line: '0B101',
+		tokens: [
+			{ startIndex: 0, type: 'number.binary.js' }
+		]
+	}],
+
+	// Regular Expressions
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '/**/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '/***/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.js' }
+		]
+	}],
+
+	[{
+		line: '5 / 3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: 'delimiter.js' }
+		]
+	}],
+
+	// Advanced regular expressions
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.js' }
+		]
+	}],
+
+	[{
+		line: '1 / 2 / x / b;',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'delimiter.js' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.js' },
+			{ startIndex: 13, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: 'x = /foo/.test(\'\')',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'regexp.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+			{ startIndex: 10, type: 'identifier.js' },
+			{ startIndex: 14, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 15, type: 'string.js' },
+			{ startIndex: 17, type: 'delimiter.parenthesis.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/g',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.js' },
+			{ startIndex: 5, type: 'keyword.other.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/gimsuy',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.js' },
+			{ startIndex: 5, type: 'keyword.other.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/q', // invalid flag
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.js' },
+			{ startIndex: 1, type: 'identifier.js' },
+			{ startIndex: 4, type: 'delimiter.js' },
+			{ startIndex: 5, type: 'identifier.js' }
+		]
+	}],
+
+	[{
+		line: 'x = 1 + f(2 / 3, /foo/)',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
+			{ startIndex: 9, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 10, type: 'number.js' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'delimiter.js' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'number.js' },
+			{ startIndex: 15, type: 'delimiter.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'regexp.js' },
+			{ startIndex: 22, type: 'delimiter.parenthesis.js' }
+		]
+	}],
+
+	[{
+		line: 'a /ads/ b;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: 'identifier.js' },
+			{ startIndex: 6, type: 'delimiter.js' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.js' },
+			{ startIndex: 9, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '1/(2/3)/2/3;',
+		tokens: [
+			{ startIndex: 0, type: 'number.js' },
+			{ startIndex: 1, type: 'delimiter.js' },
+			{ startIndex: 2, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 3, type: 'number.js' },
+			{ startIndex: 4, type: 'delimiter.js' },
+			{ startIndex: 5, type: 'number.js' },
+			{ startIndex: 6, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 7, type: 'delimiter.js' },
+			{ startIndex: 8, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+			{ startIndex: 10, type: 'number.js' },
+			{ startIndex: 11, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '{ key: 123 }',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.bracket.js' }
+		]
+	}],
+
+	[{
+		line: '[1,2,3]',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.square.js' },
+			{ startIndex: 1, type: 'number.js' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: 'number.js' },
+			{ startIndex: 4, type: 'delimiter.js' },
+			{ startIndex: 5, type: 'number.js' },
+			{ startIndex: 6, type: 'delimiter.square.js' }
+		]
+	}],
+
+	[{
+		line: 'foo(123);',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 3, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 4, type: 'number.js' },
+			{ startIndex: 7, type: 'delimiter.parenthesis.js' },
+			{ startIndex: 8, type: 'delimiter.js' }
+		]
+	}],
+
+	[{
+		line: '{a:{b:[]}}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.bracket.js' },
+			{ startIndex: 1, type: 'identifier.js' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: 'delimiter.bracket.js' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: 'delimiter.square.js' },
+			{ startIndex: 8, type: 'delimiter.bracket.js' }
+		]
+	}],
+
+	[{
+		line: 'x = "[{()}]"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.js' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'string.js' }
+		]
+	}],
+
+
+	[{
+		line: '`${5 + \'x\' + 3}a${4}`',
+		tokens: [
+			{ startIndex: 0, type: 'string.js' },
+			{ startIndex: 1, type: 'delimiter.bracket.js' },
+			{ startIndex: 3, type: 'number.js' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'string.js' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.js' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'number.js' },
+			{ startIndex: 14, type: 'delimiter.bracket.js' },
+			{ startIndex: 15, type: 'string.js' },
+			{ startIndex: 16, type: 'delimiter.bracket.js' },
+			{ startIndex: 18, type: 'number.js' },
+			{ startIndex: 19, type: 'delimiter.bracket.js' },
+			{ startIndex: 20, type: 'string.js' },
+
+		]
+	}]
+
+]);

src/javascript/javascript.ts

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { conf as tsConf, language as tsLanguage } from '../typescript/typescript';
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+export const conf: IRichLanguageConfiguration = tsConf;
+
+export const language = <ILanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.js',
+
+	keywords: [
+		'break', 'case', 'catch', 'class', 'continue', 'const',
+		'constructor', 'debugger', 'default', 'delete', 'do', 'else',
+		'export', 'extends', 'false', 'finally', 'for', 'from', 'function',
+		'get', 'if', 'import', 'in', 'instanceof', 'let', 'new', 'null',
+		'return', 'set', 'super', 'switch', 'symbol', 'this', 'throw', 'true',
+		'try', 'typeof', 'undefined', 'var', 'void', 'while', 'with', 'yield',
+		'async', 'await', 'of'
+	],
+	typeKeywords: [],
+
+	operators: tsLanguage.operators,
+	symbols: tsLanguage.symbols,
+	escapes: tsLanguage.escapes,
+	digits: tsLanguage.digits,
+	octaldigits: tsLanguage.octaldigits,
+	binarydigits: tsLanguage.binarydigits,
+	hexdigits: tsLanguage.hexdigits,
+	regexpctl: tsLanguage.regexpctl,
+	regexpesc: tsLanguage.regexpesc,
+	tokenizer: tsLanguage.tokenizer,
+};

src/kotlin/kotlin.contribution.ts

@@ -0,0 +1,15 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'kotlin',
+	extensions: ['.kt'],
+	aliases: ['Kotlin', 'kotlin'],
+	mimetypes: ['text/x-kotlin-source', 'text/x-kotlin'],
+	loader: () => import('./kotlin')
+});

src/kotlin/kotlin.test.ts

@@ -0,0 +1,671 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('kotlin', [
+	// inline reified function
+	[{
+		line: 'inline fun <reified T : Any> foo()',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.inline.kt' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'keyword.fun.kt' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.angle.kt' },
+			{ startIndex: 12, type: 'keyword.reified.kt' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'type.identifier.kt' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'delimiter.kt' },
+			{ startIndex: 23, type: '' },
+			{ startIndex: 24, type: 'type.identifier.kt' },
+			{ startIndex: 27, type: 'delimiter.angle.kt' },
+			{ startIndex: 28, type: '' },
+			{ startIndex: 29, type: 'identifier.kt' },
+			{ startIndex: 32, type: 'delimiter.parenthesis.kt' },
+		]
+	}],
+
+	// Val declaration and assignment
+	[{
+		line: 'val x: X=5',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.val.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: 'delimiter.kt' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'type.identifier.kt' },
+			{ startIndex: 8, type: 'delimiter.kt' },
+			{ startIndex: 9, type: 'number.kt' },
+		]
+	}],
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.kt' }
+		]
+	}],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[{
+		line: '/* //*/ a',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: '/almost a comment',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.kt' },
+			{ startIndex: 1, type: 'identifier.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.kt' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.kt' },
+			{ startIndex: 5, type: 'delimiter.kt' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.kt' }
+		]
+	}],
+
+	// [{
+	// 	line: 'var x = 1 // my comment // is a nice one',
+	// 	tokens: [
+	// 		{ startIndex: 0, type: 'keyword.var.kt' },
+	// 		{ startIndex: 3, type: '' },
+	// 		{ startIndex: 4, type: 'identifier.kt' },
+	// 		{ startIndex: 5, type: '' },
+	// 		{ startIndex: 6, type: 'delimiter.kt' },
+	// 		{ startIndex: 7, type: '' },
+	// 		{ startIndex: 8, type: 'number.kt' },
+	// 		{ startIndex: 9, type: '' },
+	// 		{ startIndex: 10, type: 'comment.kt' },
+	// 		{ startIndex: 12, type: '' },
+	// 		{ startIndex: 13, type: 'comment.kt' }
+	// 	]
+	// }],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* a simple comment */ 1',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.kt' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.kt' },
+		]
+	}],
+
+	[{
+		line: 'var x = /* comment */ 1; */',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.kt' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.kt' },
+			{ startIndex: 23, type: 'delimiter.kt' },
+			{ startIndex: 24, type: '' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.kt' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.kt' },
+		]
+	}],
+
+	[{
+		line: 'var x = /** start a Java Doc comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: 'and end it */ 2',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'number.kt' },
+		]
+	}],
+
+	[{
+		line: '/** start of Java Doc',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: 'a comment between without a star',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: 'end of multiline comment*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}],
+
+	// Keywords
+	[{
+		line: 'package test class Program { fun main(vararg args: String) {} } }',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.package.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.kt' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'keyword.class.kt' },
+			{ startIndex: 18, type: '' },
+			{ startIndex: 19, type: 'type.identifier.kt' },
+			{ startIndex: 26, type: '' },
+			{ startIndex: 27, type: 'delimiter.curly.kt' },
+			{ startIndex: 28, type: '' },
+			{ startIndex: 29, type: 'keyword.fun.kt' },
+			{ startIndex: 32, type: '' },
+			{ startIndex: 33, type: 'identifier.kt' },
+			{ startIndex: 37, type: 'delimiter.parenthesis.kt' },
+			{ startIndex: 38, type: 'keyword.vararg.kt' },
+			{ startIndex: 44, type: '' },
+			{ startIndex: 45, type: 'identifier.kt' },
+			{ startIndex: 49, type: 'delimiter.kt' },
+			{ startIndex: 50, type: '' },
+			{ startIndex: 51, type: 'type.identifier.kt' },
+			{ startIndex: 57, type: 'delimiter.parenthesis.kt' },
+			{ startIndex: 58, type: '' },
+			{ startIndex: 59, type: 'delimiter.curly.kt' },
+			{ startIndex: 61, type: '' },
+			{ startIndex: 62, type: 'delimiter.curly.kt' },
+			{ startIndex: 63, type: '' },
+			{ startIndex: 64, type: 'delimiter.curly.kt' }
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '0.10',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.kt' }
+		]
+	}],
+
+	[{
+		line: '0x5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.kt' }
+		]
+	}],
+
+	[{
+		line: '023L',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' }
+		]
+	}],
+
+	[{
+		line: '0123l',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' }
+		]
+	}],
+
+	[{
+		line: '05_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' }
+		]
+	}],
+
+	[{
+		line: '0b1010_0101',
+		tokens: [
+			{ startIndex: 0, type: 'number.binary.kt' }
+		]
+	}],
+
+	[{
+		line: '0B001',
+		tokens: [
+			{ startIndex: 0, type: 'number.binary.kt' }
+		]
+	}],
+
+	[{
+		line: '10e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '10f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5e-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5E3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5E-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5F',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72E3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72E3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72E-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72e3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72e3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72e-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23L',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '23l',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '0_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '5_______2',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '3_.1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '3._1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'delimiter.kt' },
+			{ startIndex: 2, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '999_99_9999_L',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 11, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 2, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0_x52',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0x_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0x52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.kt' },
+			{ startIndex: 4, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '052_',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' },
+			{ startIndex: 3, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'delimiter.kt' },
+			{ startIndex: 2, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 3, type: 'delimiter.kt' },
+			{ startIndex: 4, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.kt' }
+		]
+	}],
+
+	// single line Strings
+	[{
+		line: 'var s = "I\'m a Kotlin String"',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.kt' },
+		]
+	}],
+
+	[{
+		line: 'var s = "concatenated" + " String"',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'string.kt' },
+			{ startIndex: 22, type: '' },
+			{ startIndex: 23, type: 'delimiter.kt' },
+			{ startIndex: 24, type: '' },
+			{ startIndex: 25, type: 'string.kt' },
+		]
+	}],
+
+	[{
+		line: '"quote in a string"',
+		tokens: [
+			{ startIndex: 0, type: 'string.kt' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.kt' },
+			{ startIndex: 10, type: 'string.escape.kt' },
+			{ startIndex: 12, type: 'string.kt' },
+			{ startIndex: 18, type: 'string.escape.kt' },
+			{ startIndex: 20, type: 'string.kt' }
+		]
+	}],
+
+	[{
+		line: '"\\"',
+		tokens: [
+			{ startIndex: 0, type: 'string.invalid.kt' }
+		]
+	}],
+
+	// Annotations
+	[{
+		line: '@',
+		tokens: [
+			{ startIndex: 0, type: '' }
+		]
+	}],
+
+	[{
+		line: '@Inject',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.kt' }
+		]
+	}],
+
+	[{
+		line: '@SuppressWarnings("aString")',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.kt' },
+			{ startIndex: 17, type: 'delimiter.parenthesis.kt' },
+			{ startIndex: 18, type: 'string.kt' },
+			{ startIndex: 27, type: 'delimiter.parenthesis.kt' }
+		]
+	}],
+
+	[{
+		line: '@ AnnotationWithKeywordAfter private',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.kt' },
+			{ startIndex: 28, type: '' },
+			{ startIndex: 29, type: 'keyword.private.kt' }
+		]
+	}]
+]);
+

src/kotlin/kotlin.ts

@@ -0,0 +1,158 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')'],
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+		{ open: '<', close: '>' },
+	],
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*//\\s*(?:(?:#?region\\b)|(?:<editor-fold\\b))"),
+			end: new RegExp("^\\s*//\\s*(?:(?:#?endregion\\b)|(?:</editor-fold>))")
+		}
+	}
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.kt',
+
+	keywords: [
+		'as', 'as?', 'break', 'class', 'continue', 'do', 'else', 'false', 'for', 'fun', 'if',
+		'in', '!in', 'interface', 'is', '!is', 'null', 'object', 'package', 'return', 'super',
+		'this', 'throw', 'true', 'try', 'typealias', 'val', 'var', 'when', 'while', 'by',
+		'catch', 'constructor', 'delegate', 'dynamic', 'field', 'file', 'finally', 'get',
+		'import', 'init', 'param', 'property', 'receiver', 'set', 'setparam', 'where', 'actual',
+		'abstract','annotation', 'companion', 'const', 'crossinline', 'data', 'enum', 'expect',
+		'external', 'final', 'infix', 'inline', 'inner', 'internal', 'lateinit', 'noinline',
+		'open', 'operator', 'out', 'override', 'private', 'protected', 'public', 'reified',
+		'sealed', 'suspend', 'tailrec', 'vararg', 'field', 'it'
+	],
+
+	operators: [
+		'+', '-', '*', '/', '%', '=', '+=', '-=', '*=', '/=',
+		'%=', '++', '--', '&&', '||', '!', '==', '!=', '===',
+		'!==', '>', '<', '<=', '>=', '[', ']', '!!', '?.', '?:',
+		'::', '..', ':', '?', '->', '@', ';', '$', '_'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+
+		root: [
+			// class name highlighting
+			[/[A-Z][\w\$]*/, 'type.identifier' ],
+
+			// identifiers and keywords
+			[/[a-zA-Z_$][\w$]*/, {
+				cases: {
+					'@keywords': { token: 'keyword.$0' },
+					'@default': 'identifier'
+				}
+			}],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, {
+				cases: {
+					'@operators': 'delimiter',
+					'@default': ''
+				}
+			}],
+
+			// @ annotations.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, 'annotation'],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/0[xX](@hexdigits)[Ll]?/, 'number.hex'],
+			[/0(@octaldigits)[Ll]?/, 'number.octal'],
+			[/0[bB](@binarydigits)[Ll]?/, 'number.binary'],
+			[/(@digits)[fFdD]/, 'number.float'],
+			[/(@digits)[lL]?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
+			[/"/, 'string', '@string'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@javadoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		javadoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/, 'comment.doc.invalid'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+	},
+};

src/less/less.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'less',
 	extensions: ['.less'],
 	aliases: ['Less', 'less'],
 	mimetypes: ['text/x-less', 'text/less'],
-	loader: () => _monaco.Promise.wrap(import('./less'))
+	loader: () => import('./less')
 });

src/less/less.test.ts

@@ -7,7 +7,7 @@
 
 import { testTokenization } from '../test/testRunner';
 
-testTokenization(['less'], [
+testTokenization(['less', 'javascript'], [
 
 	// Keywords
 	[{
@@ -641,7 +641,10 @@ testTokenization(['less'], [
 			{ startIndex: 5, type: 'delimiter.less' },
 			{ startIndex: 6, type: '' },
 			{ startIndex: 7, type: 'delimiter.backtick.less' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'identifier.js' },
+			{ startIndex: 24, type: 'delimiter.parenthesis.js' },
 			{ startIndex: 26, type: 'delimiter.backtick.less' },
 			{ startIndex: 27, type: '' },
 			{ startIndex: 28, type: 'comment.less' }

src/lua/lua.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'lua',
 	extensions: ['.lua'],
 	aliases: ['Lua', 'lua'],
-	loader: () => _monaco.Promise.wrap(import('./lua'))
+	loader: () => import('./lua')
 });

src/markdown/markdown.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'markdown',
 	extensions: ['.md', '.markdown', '.mdown', '.mkdn', '.mkd', '.mdwn', '.mdtxt', '.mdtext'],
 	aliases: ['Markdown', 'markdown'],
-	loader: () => _monaco.Promise.wrap(import('./markdown'))
+	loader: () => import('./markdown')
 });

src/markdown/markdown.ts

@@ -8,23 +8,6 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-const TOKEN_HEADER_LEAD = 'keyword';
-const TOKEN_HEADER = 'keyword';
-const TOKEN_EXT_HEADER = 'keyword';
-const TOKEN_SEPARATOR = 'meta.separator';
-const TOKEN_QUOTE = 'comment';
-const TOKEN_LIST = 'keyword';
-const TOKEN_BLOCK = 'string';
-const TOKEN_BLOCK_CODE = 'variable.source';
-
-const DELIM_ASSIGN = 'delimiter.html';
-const ATTRIB_NAME = 'attribute.name.html';
-const ATTRIB_VALUE = 'string.html';
-
-function getTag(name: string) {
-	return 'tag';
-}
-
 export const conf: IRichLanguageConfiguration = {
 	comments: {
 		blockComment: ['<!--', '-->',]
@@ -75,46 +58,46 @@ export const language = <ILanguage>{
 		root: [
 
 			// headers (with #)
-			[/^(\s{0,3})(#+)((?:[^\\#]|@escapes)+)((?:#+)?)/, ['white', TOKEN_HEADER_LEAD, TOKEN_HEADER, TOKEN_HEADER]],
+			[/^(\s{0,3})(#+)((?:[^\\#]|@escapes)+)((?:#+)?)/, ['white', 'keyword', 'keyword', 'keyword']],
 
 			// headers (with =)
-			[/^\s*(=+|\-+)\s*$/, TOKEN_EXT_HEADER],
+			[/^\s*(=+|\-+)\s*$/, 'keyword'],
 
 			// headers (with ***)
-			[/^\s*((\*[ ]?)+)\s*$/, TOKEN_SEPARATOR],
+			[/^\s*((\*[ ]?)+)\s*$/, 'meta.separator'],
 
 			// quote
-			[/^\s*>+/, TOKEN_QUOTE],
+			[/^\s*>+/, 'comment'],
 
 			// list (starting with * or number)
-			[/^\s*([\*\-+:]|\d+\.)\s/, TOKEN_LIST],
+			[/^\s*([\*\-+:]|\d+\.)\s/, 'keyword'],
 
 			// code block (4 spaces indent)
-			[/^(\t|[ ]{4})[^ ].*$/, TOKEN_BLOCK],
+			[/^(\t|[ ]{4})[^ ].*$/, 'string'],
 
 			// code block (3 tilde)
-			[/^\s*~~~\s*((?:\w|[\/\-#])+)?\s*$/, { token: TOKEN_BLOCK, next: '@codeblock' }],
+			[/^\s*~~~\s*((?:\w|[\/\-#])+)?\s*$/, { token: 'string', next: '@codeblock' }],
 
 			// github style code blocks (with backticks and language)
-			[/^\s*```\s*((?:\w|[\/\-#])+)\s*$/, { token: TOKEN_BLOCK, next: '@codeblockgh', nextEmbedded: '$1' }],
+			[/^\s*```\s*((?:\w|[\/\-#])+).*$/, { token: 'string', next: '@codeblockgh', nextEmbedded: '$1' }],
 
 			// github style code blocks (with backticks but no language)
-			[/^\s*```\s*$/, { token: TOKEN_BLOCK, next: '@codeblock' }],
+			[/^\s*```\s*$/, { token: 'string', next: '@codeblock' }],
 
 			// markup within lines
 			{ include: '@linecontent' },
 		],
 
 		codeblock: [
-			[/^\s*~~~\s*$/, { token: TOKEN_BLOCK, next: '@pop' }],
-			[/^\s*```\s*$/, { token: TOKEN_BLOCK, next: '@pop' }],
-			[/.*$/, TOKEN_BLOCK_CODE],
+			[/^\s*~~~\s*$/, { token: 'string', next: '@pop' }],
+			[/^\s*```\s*$/, { token: 'string', next: '@pop' }],
+			[/.*$/, 'variable.source'],
 		],
 
 		// github style code blocks
 		codeblockgh: [
-			[/```\s*$/, { token: TOKEN_BLOCK_CODE, next: '@pop', nextEmbedded: '@pop' }],
-			[/[^`]+/, TOKEN_BLOCK_CODE],
+			[/```\s*$/, { token: 'variable.source', next: '@pop', nextEmbedded: '@pop' }],
+			[/[^`]+/, 'variable.source'],
 		],
 
 		linecontent: [
@@ -131,7 +114,7 @@ export const language = <ILanguage>{
 			[/`([^\\`]|@escapes)+`/, 'variable'],
 
 			// links
-			[/\{[^}]+\}/, 'string.target'],
+			[/\{+[^}]+\}+/, 'string.target'],
 			[/(!?\[)((?:[^\]\\]|@escapes)*)(\]\([^\)]+\))/, ['string.link', '', 'string.link']],
 			[/(!?\[)((?:[^\]\\]|@escapes)*)(\])/, 'string.link'],
 
@@ -146,14 +129,14 @@ export const language = <ILanguage>{
 		// we cannot correctly tokenize it in that mode yet.
 		html: [
 			// html tags
-			[/<(\w+)\/>/, getTag('$1')],
+			[/<(\w+)\/>/, 'tag'],
 			[/<(\w+)/, {
 				cases: {
-					'@empty': { token: getTag('$1'), next: '@tag.$1' },
-					'@default': { token: getTag('$1'), next: '@tag.$1' }
+					'@empty': { token: 'tag', next: '@tag.$1' },
+					'@default': { token: 'tag', next: '@tag.$1' }
 				}
 			}],
-			[/<\/(\w+)\s*>/, { token: getTag('$1') }],
+			[/<\/(\w+)\s*>/, { token: 'tag' }],
 
 			[/<!--/, 'comment', '@comment']
 		],
@@ -168,25 +151,25 @@ export const language = <ILanguage>{
 		// Almost full HTML tag matching, complete with embedded scripts & styles
 		tag: [
 			[/[ \t\r\n]+/, 'white'],
-			[/(type)(\s*=\s*)(")([^"]+)(")/, [ATTRIB_NAME, DELIM_ASSIGN, ATTRIB_VALUE,
-				{ token: ATTRIB_VALUE, switchTo: '@tag.$S2.$4' },
-				ATTRIB_VALUE]],
-			[/(type)(\s*=\s*)(')([^']+)(')/, [ATTRIB_NAME, DELIM_ASSIGN, ATTRIB_VALUE,
-				{ token: ATTRIB_VALUE, switchTo: '@tag.$S2.$4' },
-				ATTRIB_VALUE]],
-			[/(\w+)(\s*=\s*)("[^"]*"|'[^']*')/, [ATTRIB_NAME, DELIM_ASSIGN, ATTRIB_VALUE]],
-			[/\w+/, ATTRIB_NAME],
-			[/\/>/, getTag('$S2'), '@pop'],
+			[/(type)(\s*=\s*)(")([^"]+)(")/, ['attribute.name.html', 'delimiter.html', 'string.html',
+				{ token: 'string.html', switchTo: '@tag.$S2.$4' },
+				'string.html']],
+			[/(type)(\s*=\s*)(')([^']+)(')/, ['attribute.name.html', 'delimiter.html', 'string.html',
+				{ token: 'string.html', switchTo: '@tag.$S2.$4' },
+				'string.html']],
+			[/(\w+)(\s*=\s*)("[^"]*"|'[^']*')/, ['attribute.name.html', 'delimiter.html', 'string.html']],
+			[/\w+/, 'attribute.name.html'],
+			[/\/>/, 'tag', '@pop'],
 			[/>/, {
 				cases: {
-					'$S2==style': { token: getTag('$S2'), switchTo: 'embeddedStyle', nextEmbedded: 'text/css' },
+					'$S2==style': { token: 'tag', switchTo: 'embeddedStyle', nextEmbedded: 'text/css' },
 					'$S2==script': {
 						cases: {
-							'$S3': { token: getTag('$S2'), switchTo: 'embeddedScript', nextEmbedded: '$S3' },
-							'@default': { token: getTag('$S2'), switchTo: 'embeddedScript', nextEmbedded: 'text/javascript' }
+							'$S3': { token: 'tag', switchTo: 'embeddedScript', nextEmbedded: '$S3' },
+							'@default': { token: 'tag', switchTo: 'embeddedScript', nextEmbedded: 'text/javascript' }
 						}
 					},
-					'@default': { token: getTag('$S2'), next: '@pop' }
+					'@default': { token: 'tag', next: '@pop' }
 				}
 			}],
 		],

src/monaco.contribution.ts

@@ -17,15 +17,19 @@ import './handlebars/handlebars.contribution';
 import './html/html.contribution';
 import './ini/ini.contribution';
 import './java/java.contribution';
+import './javascript/javascript.contribution';
+import './kotlin/kotlin.contribution';
 import './less/less.contribution';
 import './lua/lua.contribution';
 import './markdown/markdown.contribution';
 import './msdax/msdax.contribution';
 import './mysql/mysql.contribution';
 import './objective-c/objective-c.contribution';
+import './pascal/pascal.contribution';
 import './pgsql/pgsql.contribution';
 import './php/php.contribution';
 import './postiats/postiats.contribution';
+import './powerquery/powerquery.contribution';
 import './powershell/powershell.contribution';
 import './pug/pug.contribution';
 import './python/python.contribution';
@@ -41,6 +45,15 @@ import './solidity/solidity.contribution';
 import './sql/sql.contribution';
 import './st/st.contribution';
 import './swift/swift.contribution';
+import './tcl/tcl.contribution';
+import './typescript/typescript.contribution';
 import './vb/vb.contribution';
 import './xml/xml.contribution';
 import './yaml/yaml.contribution';
+import './scheme/scheme.contribution';
+import './clojure/clojure.contribution';
+import './shell/shell.contribution';
+import './perl/perl.contribution';
+import './azcli/azcli.contribution';
+import './apex/apex.contribution';
+import './graphql/graphql.contribution';

src/msdax/msdax.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'msdax',
 	extensions: ['.dax', '.msdax'],
 	aliases: ['DAX', 'MSDAX'],
-	loader: () => _monaco.Promise.wrap(import('./msdax'))
+	loader: () => import('./msdax')
 });

src/mysql/mysql.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'mysql',
 	extensions: [],
 	aliases: ['MySQL', 'mysql'],
-	loader: () => _monaco.Promise.wrap(import('./mysql'))
+	loader: () => import('./mysql')
 });

src/objective-c/objective-c.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'objective-c',
 	extensions: ['.m'],
 	aliases: ['Objective-C'],
-	loader: () => _monaco.Promise.wrap(import('./objective-c'))
+	loader: () => import('./objective-c')
 });

src/objective-c/objective-c.ts

@@ -201,18 +201,18 @@ export const language = <ILanguage>{
 			[/"/, 'string.escape', '@dblStringBody']
 		],
 		stringBody: [
+			[/[^\\']+$/, 'string', '@popall'],
+			[/[^\\']+/, 'string'],
 			[/\\./, 'string'],
 			[/'/, 'string.escape', '@popall'],
-			[/.(?=.*')/, 'string'],
-			[/.*\\$/, 'string'],
-			[/.*$/, 'string', '@popall']
+			[/\\$/, 'string']
 		],
 		dblStringBody: [
+			[/[^\\"]+$/, 'string', '@popall'],
+			[/[^\\"]+/, 'string'],
 			[/\\./, 'string'],
 			[/"/, 'string.escape', '@popall'],
-			[/.(?=.*")/, 'string'],
-			[/.*\\$/, 'string'],
-			[/.*$/, 'string', '@popall']
+			[/\\$/, 'string']
 		]
 	}
 };

src/pascal/pascal.contribution.ts

@@ -0,0 +1,15 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'pascal',
+	extensions: ['.pas', '.p', '.pp'],
+	aliases: ['Pascal', 'pas'],
+	mimetypes: ['text/x-pascal-source', 'text/x-pascal'],
+	loader: () => import('./pascal')
+});

src/pascal/pascal.test.ts

@@ -0,0 +1,147 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('pascal', [
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: 'white.pascal' },
+			{ startIndex: 4, type: 'comment.pascal' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}],
+
+	// Comments - multi line (single line)
+	[{
+		line: '{}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}],
+
+	[{
+		line: '    { a comment }',
+		tokens: [
+			{ startIndex: 0, type: 'white.pascal' },
+			{ startIndex: 4, type: 'comment.pascal' }
+		]
+	}],
+
+	[{
+		line: '{ a comment }',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}],
+
+	[{
+		line: '{sticky comment}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}],
+
+	// Comments - multi line (multi line)
+	[{
+		line: '{ start of multiline comment ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal' }
+		]
+	}, {
+		line: 'a comment between curly',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal'}
+		]
+	}, {
+		line: 'end of multiline comment}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pascal'}
+		]
+	}],
+
+	// Keywords
+	[{
+		line: 'program Test; begin writeln(\'hello\'); end.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.program.pascal'},
+			{ startIndex: 7, type: 'white.pascal'},
+			{ startIndex: 8, type: 'identifier.pascal'},
+			{ startIndex: 12, type: 'delimiter.pascal'},
+			{ startIndex: 13, type: 'white.pascal'},
+			{ startIndex: 14, type: 'keyword.begin.pascal'},
+			{ startIndex: 19, type: 'white.pascal'},
+			{ startIndex: 20, type: 'identifier.pascal'},
+			{ startIndex: 27, type: 'delimiter.parenthesis.pascal'},
+			{ startIndex: 28, type: 'string.pascal'},
+			{ startIndex: 34, type: 'string.quote.pascal'},
+			{ startIndex: 35, type: 'delimiter.parenthesis.pascal'},
+			{ startIndex: 36, type: 'delimiter.pascal'},
+			{ startIndex: 37, type: 'white.pascal'},
+			{ startIndex: 38, type: 'keyword.end.pascal'},
+			{ startIndex: 41, type: 'delimiter.pascal'},
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.pascal'}
+		]
+	}],
+	[{
+		line: '0;',
+		tokens: [
+			{ startIndex: 0, type: 'number.pascal'},
+			{ startIndex: 1, type: 'delimiter.pascal'}
+		]
+	}],
+	[{
+		line: '2.4',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.pascal'}
+		]
+	}],
+	[{
+		line: '2.4;',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.pascal'},
+			{ startIndex: 3, type: 'delimiter.pascal'}
+		]
+	}],
+	[{
+		line: '$123FF',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pascal'}
+		]
+	}]
+
+]);

src/pascal/pascal.ts

@@ -0,0 +1,152 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['{', '}'],
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')'],
+		['<', '>'],
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: '\'', close: '\'' },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: '\'', close: '\'' },
+	],
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*\\{\\$REGION(\\s\\'.*\\')?\\}"),
+			end: new RegExp("^\\s*\\{\\$ENDREGION\\}")
+		}
+	}
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.pascal',
+	ignoreCase: true,
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	keywords: [
+		'absolute', 'abstract', 'all', 'and_then', 'array', 'as', 'asm',
+		'attribute', 'begin', 'bindable', 'case', 'class', 'const',
+		'contains', 'default', 'div', 'else', 'end', 'except',
+		'exports', 'external', 'far', 'file', 'finalization', 'finally',
+		'forward', 'generic', 'goto', 'if', 'implements', 'import', 'in',
+		'index', 'inherited', 'initialization', 'interrupt', 'is', 'label',
+		'library', 'mod', 'module', 'name', 'near', 'not', 'object', 'of',
+		'on', 'only', 'operator', 'or_else', 'otherwise', 'override',
+		'package', 'packed', 'pow', 'private', 'program', 'protected',
+		'public', 'published', 'interface', 'implementation', 'qualified',
+		'read', 'record', 'resident', 'requires', 'resourcestring',
+		'restricted', 'segment', 'set', 'shl', 'shr', 'specialize', 'stored',
+		'then', 'threadvar', 'to', 'try', 'type', 'unit', 'uses', 'var',
+		'view', 'virtual', 'dynamic', 'overload', 'reintroduce', 'with',
+		'write', 'xor', 'true', 'false', 'procedure', 'function',
+		'constructor','destructor','property', 'break', 'continue', 'exit',
+		'abort', 'while', 'do', 'for', 'raise', 'repeat','until'
+	],
+
+	typeKeywords: [
+		'boolean', 'double', 'byte', 'integer', 'shortint', 'char',
+		'longint', 'float', 'string'
+	],
+
+	operators: [
+		'=', '>', '<', '<=', '>=', '<>', ':', ':=', 'and', 'or',
+		'+', '-', '*', '/', '@', '&', '^', '%'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><:@\^&|+\-*\/\^%]+/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_][\w]*/, {
+				cases: {
+					'@keywords': { token: 'keyword.$0' },
+					'@default': 'identifier'
+				}
+			}],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, {
+				cases: {
+					'@operators': 'delimiter',
+					'@default': ''
+				}
+			}],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/\$[0-9a-fA-F]{1,16}/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/'([^'\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
+			[/'/, 'string', '@string'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/'/, 'string.invalid'],
+			[/\#\d+/,'string']
+		],
+
+		comment: [
+			[/[^\*\}]+/, 'comment' ],
+			//[/\(\*/,    'comment', '@push' ],    // nested comment  not allowed :-(
+			[/\}/,    'comment', '@pop'  ],
+			[/[\{]/,   'comment' ]
+		  ],
+
+		string: [
+		  [/[^\\']+/,  'string'],
+		  [/\\./,      'string.escape.invalid'],
+		  [/'/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+			[/\{/,       'comment', '@comment' ],
+			[/\/\/.*$/,    'comment'],
+		  ],
+	},
+};

src/perl/perl.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the MIT License. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'perl',
+	extensions: ['.pl'],
+	aliases: ['Perl', 'pl'],
+	loader: () => import('./perl'),
+});

src/perl/perl.test.ts

@@ -0,0 +1,476 @@
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the MIT License. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('perl', [
+	// Keywords
+	[
+		{
+			line: 'if $msg',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.perl' },
+				{ startIndex: 2, type: 'white.perl' },
+				{ startIndex: 3, type: 'variable.perl' },
+			],
+		},
+	],
+
+	// Builtins
+	[
+		{
+			line: 'log $ARGV',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.perl' },
+				{ startIndex: 3, type: 'white.perl' },
+				{ startIndex: 4, type: 'variable.predefined.perl' },
+			],
+		},
+	],
+
+	// Shebang
+	[
+		{
+			line: '#!/bin/env perl',
+			tokens: [{ startIndex: 0, type: 'metatag.perl' }],
+		},
+	],
+
+	// Comments - single line
+	[
+		{
+			line: '#',
+			tokens: [{ startIndex: 0, type: 'comment.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '    # a comment',
+			tokens: [
+				{ startIndex: 0, type: 'white.perl' },
+				{ startIndex: 4, type: 'comment.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '# a comment',
+			tokens: [{ startIndex: 0, type: 'comment.perl' }],
+		},
+	],
+
+	// number
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '0.0',
+			tokens: [{ startIndex: 0, type: 'number.float.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '1.72e-3',
+			tokens: [{ startIndex: 0, type: 'number.float.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.perl' },
+				{ startIndex: 1, type: 'operators.perl' },
+				{ startIndex: 2, type: 'number.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.perl' },
+				{ startIndex: 3, type: 'operators.perl' },
+				{ startIndex: 4, type: 'number.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.perl' },
+				{ startIndex: 1, type: 'white.perl' },
+				{ startIndex: 2, type: 'operators.perl' },
+				{ startIndex: 3, type: 'white.perl' },
+				{ startIndex: 4, type: 'number.perl' },
+			],
+		},
+	],
+
+	// Strings
+
+	// Double quoted string
+	[
+		{
+			line: '"string"',
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+	],
+
+	[
+		{
+			line: '"test $foo"',
+			tokens: [
+				{ startIndex: 0, type: 'string.perl' },
+				{ startIndex: 6, type: 'variable.perl' },
+				{ startIndex: 10, type: 'string.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '"test',
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+		{
+			line: '',
+			tokens: [],
+		},
+		{
+			line: 'string $foo string2"',
+			tokens: [
+				{ startIndex: 0, type: 'string.perl' },
+				{ startIndex: 7, type: 'variable.perl' },
+				{ startIndex: 11, type: 'string.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '"string\\t"',
+			tokens: [
+				{ startIndex: 0, type: 'string.perl' },
+				{
+					startIndex: 7,
+					type: 'string.escape.perl',
+				},
+				{ startIndex: 9, type: 'string.perl' },
+			],
+		},
+	],
+
+	// Single quoted string
+	[
+		{
+			line: "'string'",
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+	],
+
+	[
+		{
+			line: "'test $foo'",
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+	],
+
+	[
+		{
+			line: "'test",
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+		{
+			line: '',
+			tokens: [],
+		},
+		{
+			line: "string $foo string2'",
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+	],
+
+	[
+		{
+			line: "'string\\t'",
+			tokens: [{ startIndex: 0, type: 'string.perl' }],
+		},
+	],
+
+	[
+		{
+			line: "'string\\'string2'",
+			tokens: [
+				{ startIndex: 0, type: 'string.perl' },
+				{
+					startIndex: 7,
+					type: 'string.escape.perl',
+				},
+				{ startIndex: 9, type: 'string.perl' },
+			],
+		},
+	],
+
+	// Variables
+	[
+		{
+			line: '$msg $_ $1',
+			tokens: [
+				{ startIndex: 0, type: 'variable.perl' },
+				{ startIndex: 4, type: 'white.perl' },
+				{ startIndex: 5, type: 'variable.predefined.perl' },
+				{ startIndex: 7, type: 'white.perl' },
+				{ startIndex: 8, type: 'variable.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '@array1 @array2',
+			tokens: [
+				{ startIndex: 0, type: 'variable.perl' },
+				{ startIndex: 7, type: 'white.perl' },
+				{
+					startIndex: 8,
+					type: 'variable.perl',
+				},
+			],
+		},
+	],
+
+	[
+		{
+			line: '%var1 %var2',
+			tokens: [
+				{ startIndex: 0, type: 'variable.perl' },
+				{
+					startIndex: 5,
+					type: 'white.perl',
+				},
+				{
+					startIndex: 6,
+					type: 'variable.perl',
+				},
+			],
+		},
+	],
+
+	// RegExp
+	[
+		{
+			line: '/abc/',
+			tokens: [{ startIndex: 0, type: 'regexp.perl' }],
+		},
+	],
+
+	[
+		{
+			line: 'm/abc/',
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delim.perl' },
+				{ startIndex: 2, type: 'regexp.perl' },
+				{ startIndex: 5, type: 'regexp.delim.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: 'm/[abc]+/e',
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delim.perl' },
+				{ startIndex: 2, type: 'regexp.perl' },
+				{ startIndex: 8, type: 'regexp.delim.perl' },
+        { startIndex: 9, type: 'regexp.modifier.perl' },
+			],
+		},
+	],
+
+	// Operators
+	[
+		{
+			line: '$a + $b',
+			tokens: [
+				{ startIndex: 0, type: 'variable.predefined.perl' },
+				{
+					startIndex: 2,
+					type: 'white.perl',
+				},
+				{
+					startIndex: 3,
+					type: 'operators.perl',
+				},
+				{ startIndex: 4, type: 'white.perl' },
+				{ startIndex: 5, type: 'variable.predefined.perl' },
+			],
+		},
+	],
+
+	// Embedded Doc
+	[
+		{
+			line: '=begin',
+			tokens: [
+				{
+					startIndex: 0,
+					type: 'comment.doc.perl',
+				},
+			],
+		},
+		{
+			line: 'this is my doc',
+			tokens: [
+				{
+					startIndex: 0,
+					type: 'comment.doc.perl',
+				},
+			],
+		},
+		{
+			line: '=cut',
+			tokens: [{ startIndex: 0, type: 'type.identifier.perl' }],
+		},
+	],
+
+	// Here Doc
+	[
+		{
+			line: '<< HTML',
+			tokens: [{ startIndex: 0, type: 'string.heredoc.delimiter.perl' }],
+		},
+		{
+			line: 'test here doc',
+			tokens: [
+				{
+					startIndex: 0,
+					type: 'string.heredoc.perl',
+				},
+			],
+		},
+		{
+			line: 'HTML',
+			tokens: [{ startIndex: 0, type: 'string.heredoc.delimiter.perl' }],
+		},
+		{
+			line: 'my $msg',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.perl' },
+				{
+					startIndex: 2,
+					type: 'white.perl',
+				},
+				{ startIndex: 3, type: 'variable.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: '<<"HTML"',
+			tokens: [{ startIndex: 0, type: 'string.heredoc.delimiter.perl' }],
+		},
+		{
+			line: 'test here doc',
+			tokens: [
+				{
+					startIndex: 0,
+					type: 'string.heredoc.perl',
+				},
+			],
+		},
+		{
+			line: 'HTML',
+			tokens: [{ startIndex: 0, type: 'string.heredoc.delimiter.perl' }],
+		},
+		{
+			line: 'my $msg',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.perl' },
+				{
+					startIndex: 2,
+					type: 'white.perl',
+				},
+				{ startIndex: 3, type: 'variable.perl' },
+			],
+		},
+	],
+
+  // Quoted constructs
+	[
+		{
+			line: "m!can't!",
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delim.perl' },
+				{ startIndex: 2, type: 'regexp.perl' },
+				{ startIndex: 7, type: 'regexp.delim.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: 'q XfooX',
+			tokens: [
+				{ startIndex: 0, type: 'string.delim.perl' },
+				{ startIndex: 3, type: 'string.perl' },
+				{ startIndex: 6, type: 'string.delim.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: 'qq(test $foo)',
+			tokens: [
+				{ startIndex: 0, type: 'string.delim.perl' },
+				{ startIndex: 3, type: 'string.perl' },
+				{ startIndex: 8, type: 'variable.perl' },
+        { startIndex: 12, type: 'string.delim.perl' },
+			],
+		},
+	],
+]);

src/perl/perl.ts

@@ -0,0 +1,657 @@
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the MIT License. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+	},
+	brackets: [['{', '}'], ['[', ']'], ['(', ')']],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '`', close: '`' },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '`', close: '`' },
+	],
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.perl',
+
+	brackets: [
+		{ token: 'delimiter.bracket', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+	],
+
+	// https://learn.perl.org/docs/keywords.html
+
+	// Perl syntax
+	keywords: [
+		'__DATA__',
+		'else',
+		'lock',
+		'__END__',
+		'elsif',
+		'lt',
+		'__FILE__',
+		'eq',
+		'__LINE__',
+		'exp',
+		'ne',
+		'sub',
+		'__PACKAGE__',
+		'for',
+		'no',
+		'and',
+		'foreach',
+		'or',
+		'unless',
+		'cmp',
+		'ge',
+		'package',
+		'until',
+		'continue',
+		'gt',
+		'while',
+		'CORE',
+		'if',
+		'xor',
+		'do',
+		'le',
+
+		'__DIE__',
+		'__WARN__',
+	],
+
+	// Perl functions
+	builtinFunctions: [
+		'-A',
+		'END',
+		'length',
+		'setpgrp',
+		'-B',
+		'endgrent',
+		'link',
+		'setpriority',
+		'-b',
+		'endhostent',
+		'listen',
+		'setprotoent',
+		'-C',
+		'endnetent',
+		'local',
+		'setpwent',
+		'-c',
+		'endprotoent',
+		'localtime',
+		'setservent',
+		'-d',
+		'endpwent',
+		'log',
+		'setsockopt',
+		'-e',
+		'endservent',
+		'lstat',
+		'shift',
+		'-f',
+		'eof',
+		'map',
+		'shmctl',
+		'-g',
+		'eval',
+		'mkdir',
+		'shmget',
+		'-k',
+		'exec',
+		'msgctl',
+		'shmread',
+		'-l',
+		'exists',
+		'msgget',
+		'shmwrite',
+		'-M',
+		'exit',
+		'msgrcv',
+		'shutdown',
+		'-O',
+		'fcntl',
+		'msgsnd',
+		'sin',
+		'-o',
+		'fileno',
+		'my',
+		'sleep',
+		'-p',
+		'flock',
+		'next',
+		'socket',
+		'-r',
+		'fork',
+		'not',
+		'socketpair',
+		'-R',
+		'format',
+		'oct',
+		'sort',
+		'-S',
+		'formline',
+		'open',
+		'splice',
+		'-s',
+		'getc',
+		'opendir',
+		'split',
+		'-T',
+		'getgrent',
+		'ord',
+		'sprintf',
+		'-t',
+		'getgrgid',
+		'our',
+		'sqrt',
+		'-u',
+		'getgrnam',
+		'pack',
+		'srand',
+		'-w',
+		'gethostbyaddr',
+		'pipe',
+		'stat',
+		'-W',
+		'gethostbyname',
+		'pop',
+		'state',
+		'-X',
+		'gethostent',
+		'pos',
+		'study',
+		'-x',
+		'getlogin',
+		'print',
+		'substr',
+		'-z',
+		'getnetbyaddr',
+		'printf',
+		'symlink',
+		'abs',
+		'getnetbyname',
+		'prototype',
+		'syscall',
+		'accept',
+		'getnetent',
+		'push',
+		'sysopen',
+		'alarm',
+		'getpeername',
+		'quotemeta',
+		'sysread',
+		'atan2',
+		'getpgrp',
+		'rand',
+		'sysseek',
+		'AUTOLOAD',
+		'getppid',
+		'read',
+		'system',
+		'BEGIN',
+		'getpriority',
+		'readdir',
+		'syswrite',
+		'bind',
+		'getprotobyname',
+		'readline',
+		'tell',
+		'binmode',
+		'getprotobynumber',
+		'readlink',
+		'telldir',
+		'bless',
+		'getprotoent',
+		'readpipe',
+		'tie',
+		'break',
+		'getpwent',
+		'recv',
+		'tied',
+		'caller',
+		'getpwnam',
+		'redo',
+		'time',
+		'chdir',
+		'getpwuid',
+		'ref',
+		'times',
+		'CHECK',
+		'getservbyname',
+		'rename',
+		'truncate',
+		'chmod',
+		'getservbyport',
+		'require',
+		'uc',
+		'chomp',
+		'getservent',
+		'reset',
+		'ucfirst',
+		'chop',
+		'getsockname',
+		'return',
+		'umask',
+		'chown',
+		'getsockopt',
+		'reverse',
+		'undef',
+		'chr',
+		'glob',
+		'rewinddir',
+		'UNITCHECK',
+		'chroot',
+		'gmtime',
+		'rindex',
+		'unlink',
+		'close',
+		'goto',
+		'rmdir',
+		'unpack',
+		'closedir',
+		'grep',
+		'say',
+		'unshift',
+		'connect',
+		'hex',
+		'scalar',
+		'untie',
+		'cos',
+		'index',
+		'seek',
+		'use',
+		'crypt',
+		'INIT',
+		'seekdir',
+		'utime',
+		'dbmclose',
+		'int',
+		'select',
+		'values',
+		'dbmopen',
+		'ioctl',
+		'semctl',
+		'vec',
+		'defined',
+		'join',
+		'semget',
+		'wait',
+		'delete',
+		'keys',
+		'semop',
+		'waitpid',
+		'DESTROY',
+		'kill',
+		'send',
+		'wantarray',
+		'die',
+		'last',
+		'setgrent',
+		'warn',
+		'dump',
+		'lc',
+		'sethostent',
+		'write',
+		'each',
+		'lcfirst',
+		'setnetent',
+	],
+
+	// File handlers
+	builtinFileHandlers: ['ARGV', 'STDERR', 'STDOUT', 'ARGVOUT', 'STDIN', 'ENV'],
+
+	// Perl variables
+	builtinVariables: [
+		'$!',
+		'$^RE_TRIE_MAXBUF',
+		'$LAST_REGEXP_CODE_RESULT',
+		'$"',
+		'$^S',
+		'$LIST_SEPARATOR',
+		'$#',
+		'$^T',
+		'$MATCH',
+		'$$',
+		'$^TAINT',
+		'$MULTILINE_MATCHING',
+		'$%',
+		'$^UNICODE',
+		'$NR',
+		'$&',
+		'$^UTF8LOCALE',
+		'$OFMT',
+		"$'",
+		'$^V',
+		'$OFS',
+		'$(',
+		'$^W',
+		'$ORS',
+		'$)',
+		'$^WARNING_BITS',
+		'$OS_ERROR',
+		'$*',
+		'$^WIDE_SYSTEM_CALLS',
+		'$OSNAME',
+		'$+',
+		'$^X',
+		'$OUTPUT_AUTO_FLUSH',
+		'$,',
+		'$_',
+		'$OUTPUT_FIELD_SEPARATOR',
+		'$-',
+		'$`',
+		'$OUTPUT_RECORD_SEPARATOR',
+		'$.',
+		'$a',
+		'$PERL_VERSION',
+		'$/',
+		'$ACCUMULATOR',
+		'$PERLDB',
+		'$0',
+		'$ARG',
+		'$PID',
+		'$:',
+		'$ARGV',
+		'$POSTMATCH',
+		'$;',
+		'$b',
+		'$PREMATCH',
+		'$<',
+		'$BASETIME',
+		'$PROCESS_ID',
+		'$=',
+		'$CHILD_ERROR',
+		'$PROGRAM_NAME',
+		'$>',
+		'$COMPILING',
+		'$REAL_GROUP_ID',
+		'$?',
+		'$DEBUGGING',
+		'$REAL_USER_ID',
+		'$@',
+		'$EFFECTIVE_GROUP_ID',
+		'$RS',
+		'$[',
+		'$EFFECTIVE_USER_ID',
+		'$SUBSCRIPT_SEPARATOR',
+		'$\\',
+		'$EGID',
+		'$SUBSEP',
+		'$]',
+		'$ERRNO',
+		'$SYSTEM_FD_MAX',
+		'$^',
+		'$EUID',
+		'$UID',
+		'$^A',
+		'$EVAL_ERROR',
+		'$WARNING',
+		'$^C',
+		'$EXCEPTIONS_BEING_CAUGHT',
+		'$|',
+		'$^CHILD_ERROR_NATIVE',
+		'$EXECUTABLE_NAME',
+		'$~',
+		'$^D',
+		'$EXTENDED_OS_ERROR',
+		'%!',
+		'$^E',
+		'$FORMAT_FORMFEED',
+		'%^H',
+		'$^ENCODING',
+		'$FORMAT_LINE_BREAK_CHARACTERS',
+		'%ENV',
+		'$^F',
+		'$FORMAT_LINES_LEFT',
+		'%INC',
+		'$^H',
+		'$FORMAT_LINES_PER_PAGE',
+		'%OVERLOAD',
+		'$^I',
+		'$FORMAT_NAME',
+		'%SIG',
+		'$^L',
+		'$FORMAT_PAGE_NUMBER',
+		'@+',
+		'$^M',
+		'$FORMAT_TOP_NAME',
+		'@-',
+		'$^N',
+		'$GID',
+		'@_',
+		'$^O',
+		'$INPLACE_EDIT',
+		'@ARGV',
+		'$^OPEN',
+		'$INPUT_LINE_NUMBER',
+		'@INC',
+		'$^P',
+		'$INPUT_RECORD_SEPARATOR',
+		'@LAST_MATCH_START',
+		'$^R',
+		'$LAST_MATCH_END',
+		'$^RE_DEBUG_FLAGS',
+		'$LAST_PAREN_MATCH',
+	],
+
+	// operators
+	symbols: /[:+\-\^*$&%@=<>!?|\/~\.]/,
+	quoteLikeOps: ['qr', 'm', 's', 'q', 'qq', 'qx', 'qw', 'tr', 'y'],
+
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			{ include: '@whitespace' },
+
+			[
+				/[a-zA-Z\-_][\w\-_]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@builtinFunctions': 'type.identifier',
+						'@builtinFileHandlers': 'variable.predefined',
+						'@quoteLikeOps': { token: '@rematch', next: 'quotedConstructs' },
+						'@default': '',
+					},
+				},
+			],
+
+			// Perl variables
+			[
+				/[\$@%][*@#?\+\-\$!\w\\\^><~:;\.]+/,
+				{
+					cases: {
+						'@builtinVariables': 'variable.predefined',
+						'@default': 'variable',
+					},
+				},
+			],
+
+			{ include: '@strings' },
+			{ include: '@dblStrings' },
+
+			// Perl Doc
+			{ include: '@perldoc' },
+
+			// Here Doc
+			{ include: '@heredoc' },
+
+			[/[{}\[\]()]/, '@brackets'],
+
+			// RegExp
+			[
+				/[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/,
+				'regexp',
+			],
+
+			[/@symbols/, 'operators'],
+
+			{ include: '@numbers' },
+
+			[/[,;]/, 'delimiter'],
+		],
+
+		whitespace: [
+			[/\s+/, 'white'],
+			[/(^#!.*$)/, 'metatag'],
+			[/(^#.*$)/, 'comment'],
+		],
+
+		numbers: [
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+/, 'number'],
+		],
+
+		// Single quote string
+		strings: [[/'/, 'string', '@stringBody']],
+
+		stringBody: [
+			[/'/, 'string', '@popall'],
+			[/\\'/, 'string.escape'],
+			[/./, 'string'],
+		],
+
+		// Double quote string
+		dblStrings: [[/"/, 'string', '@dblStringBody']],
+
+		dblStringBody: [
+			[/"/, 'string', '@popall'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			{ include: '@variables' },
+			[/./, 'string'],
+		],
+
+		// Quoted constructs
+		// Percent strings in Ruby are similar to quote-like operators in Perl.
+		// This is adapted from pstrings in ../ruby/ruby.ts.
+		quotedConstructs: [
+			[/(q|qw|tr|y)\s*\(/, { token: 'string.delim', switchTo: '@qstring.(.)' }],
+			[/(q|qw|tr|y)\s*\[/, { token: 'string.delim', switchTo: '@qstring.[.]' }],
+			[/(q|qw|tr|y)\s*\{/, { token: 'string.delim', switchTo: '@qstring.{.}' }],
+			[/(q|qw|tr|y)\s*</, { token: 'string.delim', switchTo: '@qstring.<.>' }],
+			[/(q|qw|tr|y)#/, { token: 'string.delim', switchTo: '@qstring.#.#' }],
+			[/(q|qw|tr|y)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }],
+			[/(q|qw|tr|y)\s+(\w)/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }],
+
+			[/(qr|m|s)\s*\(/, { token: 'regexp.delim', switchTo: '@qregexp.(.)' }],
+			[/(qr|m|s)\s*\[/, { token: 'regexp.delim', switchTo: '@qregexp.[.]' }],
+			[/(qr|m|s)\s*\{/, { token: 'regexp.delim', switchTo: '@qregexp.{.}' }],
+			[/(qr|m|s)\s*</, { token: 'regexp.delim', switchTo: '@qregexp.<.>' }],
+			[/(qr|m|s)#/, { token: 'regexp.delim', switchTo: '@qregexp.#.#' }],
+			[/(qr|m|s)\s*([^A-Za-z0-9_#\s])/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }],
+			[/(qr|m|s)\s+(\w)/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }],
+
+			[/(qq|qx)\s*\(/, { token: 'string.delim', switchTo: '@qqstring.(.)' }],
+			[/(qq|qx)\s*\[/, { token: 'string.delim', switchTo: '@qqstring.[.]' }],
+			[/(qq|qx)\s*\{/, { token: 'string.delim', switchTo: '@qqstring.{.}' }],
+			[/(qq|qx)\s*</, { token: 'string.delim', switchTo: '@qqstring.<.>' }],
+			[/(qq|qx)#/, { token: 'string.delim', switchTo: '@qqstring.#.#' }],
+			[/(qq|qx)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }],
+			[/(qq|qx)\s+(\w)/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }],
+		],
+
+		// Non-expanded quoted string
+		// qstring<open>.<close>
+		//  open = open delimiter
+		//  close = close delimiter
+		qstring: [
+			[/\\./, 'string.escape'],
+			[/./, {
+				cases: {
+					'$#==$S3': { token: 'string.delim', next: '@pop' },
+					'$#==$S2': { token: 'string.delim', next: '@push' }, // nested delimiters
+					'@default': 'string'
+				}
+			}],
+		],
+
+    // Quoted regexp
+		// qregexp.<open>.<close>
+		//  open = open delimiter
+		//  close = close delimiter
+		qregexp: [
+			{ include: '@variables' },
+			[/\\./, 'regexp.escape'],
+			[/./, {
+				cases: {
+					'$#==$S3': { token: 'regexp.delim', next: '@regexpModifiers' },
+					'$#==$S2': { token: 'regexp.delim', next: '@push' }, // nested delimiters
+					'@default': 'regexp'
+				}
+			}],
+		],
+
+    regexpModifiers: [
+			[/[msixpodualngcer]+/, { token: 'regexp.modifier', next: '@popall' }],
+		],
+
+		// Expanded quoted string
+		// qqstring.<open>.<close>
+		//  open = open delimiter
+		//  close = close delimiter
+		qqstring: [
+			{ include: '@variables' },
+			{ include: '@qstring' },
+		],
+
+		heredoc: [
+			[
+				/<<\s*['"`]?([\w\-]+)['"`]?/,
+				{ token: 'string.heredoc.delimiter', next: '@heredocBody.$1' },
+			],
+		],
+
+		heredocBody: [
+			[
+				/^([\w\-]+)$/,
+				{
+					cases: {
+						'$1==$S2': [{ token: 'string.heredoc.delimiter', next: '@popall' }],
+						'@default': 'string.heredoc',
+					},
+				},
+			],
+			[/./, 'string.heredoc'],
+		],
+
+		perldoc: [[/^=\w/, 'comment.doc', '@perldocBody']],
+
+		perldocBody: [
+			[/^=cut\b/, 'type.identifier', '@popall'],
+			[/./, 'comment.doc'],
+		],
+
+		variables: [
+			[/\$\w+/, 'variable'], // scalar
+			[/@\w+/, 'variable'], // array
+			[/%\w+/, 'variable'], // key/value
+		],
+	},
+};

src/pgsql/pgsql.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'pgsql',
 	extensions: [],
 	aliases: ['PostgreSQL', 'postgres', 'pg', 'postgre'],
-	loader: () => _monaco.Promise.wrap(import('./pgsql'))
+	loader: () => import('./pgsql')
 });

src/php/php.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'php',
 	extensions: ['.php', '.php4', '.php5', '.phtml', '.ctp'],
 	aliases: ['PHP', 'php'],
 	mimetypes: ['application/x-php'],
-	loader: () => _monaco.Promise.wrap(import('./php'))
+	loader: () => import('./php')
 });

src/php/php.test.ts

@@ -7,7 +7,7 @@
 
 import { testTokenization } from '../test/testRunner';
 
-testTokenization(['php', 'css'], [
+testTokenization(['php', 'css', 'javascript'], [
 	// Bug 13596:[ErrorTelemetry] Stream did not advance while tokenizing. Mode id is php (stuck)
 	// We're testing the fact that tokenize does not throw
 	[
@@ -1863,7 +1863,13 @@ testTokenization(['php', 'css'], [
 			// { startIndex:5, type: 'delimiter.html' },
 			{ startIndex: 6, type: 'tag.html' },
 			{ startIndex: 12, type: 'delimiter.html' },
-			{ startIndex: 13, type: '' },
+			{ startIndex: 13, type: 'keyword.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'identifier.js' },
+			{ startIndex: 18, type: 'delimiter.js' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'number.js' },
+			{ startIndex: 22, type: 'delimiter.js' },
 			{ startIndex: 23, type: 'delimiter.html' },
 			{ startIndex: 25, type: 'tag.html' },
 			{ startIndex: 31, type: 'delimiter.html' },
@@ -1888,7 +1894,13 @@ testTokenization(['php', 'css'], [
 			// { startIndex:5, type: 'delimiter.html' },
 			{ startIndex: 6, type: 'tag.html' },
 			{ startIndex: 12, type: 'delimiter.html' },
-			{ startIndex: 13, type: '' },
+			{ startIndex: 13, type: 'keyword.js' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'identifier.js' },
+			{ startIndex: 18, type: 'delimiter.js' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'number.js' },
+			{ startIndex: 22, type: 'delimiter.js' },
 			{ startIndex: 23, type: 'delimiter.html' },
 			{ startIndex: 25, type: 'tag.html' },
 			{ startIndex: 31, type: 'delimiter.html' },
@@ -1900,7 +1912,13 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 42, type: 'delimiter.html' },
 			{ startIndex: 43, type: 'tag.html' },
 			{ startIndex: 49, type: 'delimiter.html' },
-			{ startIndex: 50, type: '' },
+			{ startIndex: 50, type: 'keyword.js' },
+			{ startIndex: 53, type: '' },
+			{ startIndex: 54, type: 'identifier.js' },
+			{ startIndex: 55, type: 'delimiter.js' },
+			{ startIndex: 56, type: '' },
+			{ startIndex: 57, type: 'number.js' },
+			{ startIndex: 59, type: 'delimiter.js' },
 			{ startIndex: 60, type: 'delimiter.html' },
 			{ startIndex: 62, type: 'tag.html' },
 			{ startIndex: 68, type: 'delimiter.html' }
@@ -1969,12 +1987,28 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 73, type: 'delimiter.html' },
 			{ startIndex: 74, type: 'tag.html' },
 			{ startIndex: 80, type: 'delimiter.html' },
-			{ startIndex: 81, type: '' },
+			{ startIndex: 81, type: 'keyword.js' },
+			{ startIndex: 84, type: '' },
+			{ startIndex: 85, type: 'identifier.js' },
+			{ startIndex: 86, type: '' },
+			{ startIndex: 87, type: 'delimiter.js' },
+			{ startIndex: 88, type: '' },
+			{ startIndex: 89, type: 'number.js' },
+			{ startIndex: 90, type: 'delimiter.js' },
+			{ startIndex: 91, type: 'comment.js' },
 			{ startIndex: 94, type: 'metatag.php' },
 			{ startIndex: 97, type: 'string.php' },
 			{ startIndex: 109, type: 'comment.php' },
 			{ startIndex: 122, type: 'metatag.php' },
 			{ startIndex: 124, type: '' },
+			{ startIndex: 127, type: 'keyword.js' },
+			{ startIndex: 130, type: '' },
+			{ startIndex: 131, type: 'identifier.js' },
+			{ startIndex: 132, type: '' },
+			{ startIndex: 133, type: 'delimiter.js' },
+			{ startIndex: 134, type: '' },
+			{ startIndex: 135, type: 'number.js' },
+			{ startIndex: 136, type: 'delimiter.js' },
 			{ startIndex: 137, type: 'delimiter.html' },
 			{ startIndex: 139, type: 'tag.html' },
 			{ startIndex: 145, type: 'delimiter.html' },
@@ -2010,7 +2044,7 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'comment.js' },
 			{ startIndex: 10, type: 'metatag.php' }
 		]
 	}],
@@ -2021,13 +2055,13 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: 'delimiter.html' },
-			{ startIndex: 8, type: '' },
+			{ startIndex: 8, type: 'string.invalid.js' },
 			{ startIndex: 9, type: 'metatag.php' },
 			{ startIndex: 14, type: 'number.php' },
 			{ startIndex: 15, type: 'delimiter.php' },
 			{ startIndex: 16, type: 'number.php' },
 			{ startIndex: 17, type: 'metatag.php' },
-			{ startIndex: 19, type: '' }
+			{ startIndex: 19, type: 'string.invalid.js' }
 		]
 	}],
 
@@ -2042,5 +2076,23 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 17, type: '' },
 			{ startIndex: 18, type: 'metatag.php' }
 		]
+	}],
+
+	[{
+		line: '<?php',
+		tokens: [
+			{ startIndex: 0, type: 'metatag.php' },
+		]
+	}, {
+		line: '  //',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 2, type: 'comment.php' },
+		]
+	}, {
+		line: 'if',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.php' },
+		]
 	}]
 ]);

src/php/php.ts

@@ -225,8 +225,8 @@ export const language = <ILanguage>{
 			[/[ \t\r\n]+/],
 
 			// comments
-			[/#/, 'comment.php', '@phpLineComment'],
-			[/\/\//, 'comment.php', '@phpLineComment'],
+			[/(#|\/\/)$/, 'comment.php'],
+			[/(#|\/\/)/, 'comment.php', '@phpLineComment'],
 
 			// block comments
 			[/\/\*/, 'comment.php', '@phpComment'],

src/postiats/postiats.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'postiats',
 	extensions: ['.dats', '.sats', '.hats'],
 	aliases: ['ATS', 'ATS/Postiats'],
-	loader: () => _monaco.Promise.wrap(import('./postiats'))
+	loader: () => import('./postiats')
 });

src/powerquery/powerquery.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'powerquery',
+	extensions: ['.pq', '.pqm'],
+	aliases: ['PQ', 'M', 'Power Query', 'Power Query M'],
+	loader: () => import('./powerquery')
+});

src/powerquery/powerquery.test.ts

@@ -0,0 +1,374 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('powerquery', [
+	// Comments
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '    // a comment */',
+		tokens: [
+			{ startIndex: 0, type: 'white.pq' },
+			{ startIndex: 4, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '// /* #A */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '/*ABCD12$!()\\u000D%%%%%*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '42 /* + 45 */ /*',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' },
+			{ startIndex: 2, type: 'white.pq' },
+			{ startIndex: 3, type: 'comment.pq' },
+			{ startIndex: 13, type: 'white.pq' },
+			{ startIndex: 14, type: 'comment.pq' }
+		]
+	}],
+
+	[{
+		line: '/* //*/ a',
+		tokens: [
+			{ startIndex: 0, type: 'comment.pq' },
+			{ startIndex: 7, type: 'white.pq' },
+			{ startIndex: 8, type: 'identifier.pq' }
+		]
+	}],
+
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' },
+			{ startIndex: 1, type: 'white.pq' },
+			{ startIndex: 2, type: 'operators.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'number.pq' },
+			{ startIndex: 5, type: 'delimiter.pq' },
+			{ startIndex: 6, type: 'white.pq' },
+			{ startIndex: 7, type: 'comment.pq' }
+		]
+	}],
+
+	// Quoted Identifiers
+	[{
+		line: '#"Change Types"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.pq' }
+		]
+	}],
+
+	[{
+		line: '#"A  B" = 1+2,',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.pq' },
+			{ startIndex: 7, type: 'white.pq' },
+			{ startIndex: 8, type: 'operators.pq' },
+			{ startIndex: 9, type: 'white.pq' },
+			{ startIndex: 10, type: 'number.pq' },
+			{ startIndex: 11, type: 'operators.pq' },
+			{ startIndex: 12, type: 'number.pq' },
+			{ startIndex: 13, type: 'delimiter.pq' },
+		]
+	}],
+
+	[{
+		line: 'a = #"escap ed"+ 1',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 1, type: 'white.pq' },
+			{ startIndex: 2, type: 'operators.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'identifier.quote.pq' },
+			{ startIndex: 15, type: 'operators.pq' },
+			{ startIndex: 16, type: 'white.pq' },
+			{ startIndex: 17, type: 'number.pq' }
+		]
+	}],
+
+	// Number formats
+	[{
+		line: '0Xabc',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pq' }
+		]
+	}],
+
+	[{
+		line: '0xA',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pq' }
+		]
+	}],
+
+	[{
+		line: '1e1',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' }
+		]
+	}],
+
+	[{
+		line: '5 / 1.2e+2 + 0x1234abc',
+		tokens: [
+			{ startIndex: 0, type: 'number.pq' },
+			{ startIndex: 1, type: 'white.pq' },
+			{ startIndex: 2, type: 'operators.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'number.float.pq' },
+			{ startIndex: 10, type: 'white.pq' },
+			{ startIndex: 11, type: 'operators.pq' },
+			{ startIndex: 12, type: 'white.pq'},
+			{ startIndex: 13, type: 'number.hex.pq'}
+		]
+	}],
+
+	[{
+		line: '0xb *(.2)',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'operators.pq' },
+			{ startIndex: 5, type: 'delimiter.parenthesis.pq' },
+			{ startIndex: 6, type: 'number.float.pq' },
+			{ startIndex: 8, type: 'delimiter.parenthesis.pq' }
+		]
+	}],
+
+	[{
+		line: '1.23e34+1.2e-2-.3e2',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.pq' },
+			{ startIndex: 7, type: 'operators.pq' },
+			{ startIndex: 8, type: 'number.float.pq' },
+			{ startIndex: 14, type: 'operators.pq' },
+			{ startIndex: 15, type: 'number.float.pq' }
+		]
+	}],
+
+	// strings
+	[{
+		line: '  "string"',
+		tokens: [
+			{ startIndex: 0, type: 'white.pq' },
+			{ startIndex: 2, type: 'string.pq' }
+		]
+	}],
+
+	[{
+		line: '"string" & "another"',
+		tokens: [
+			{ startIndex: 0, type: 'string.pq' },
+			{ startIndex: 8, type: 'white.pq' },
+			{ startIndex: 9, type: 'operators.pq' },
+			{ startIndex: 10, type: 'white.pq' },
+			{ startIndex: 11, type: 'string.pq' }
+		]
+	}],
+
+	[{
+		line: '"with  ""escaped "" \'text',
+		tokens: [
+			{ startIndex: 0, type: 'string.pq' },
+			{ startIndex: 7, type: 'string.escape.pq' },
+			{ startIndex: 9, type: 'string.pq' },
+			{ startIndex: 17, type: 'string.escape.pq' },
+			{ startIndex: 19, type: 'string.pq' }
+		]
+	}],
+
+	// built-in keywords/identifiers
+	[{
+		line: 'And as Each each _',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 3, type: 'white.pq' },
+			{ startIndex: 4, type: 'keyword.pq' },
+			{ startIndex: 6, type: 'white.pq' },
+			{ startIndex: 7, type: 'identifier.pq' },
+			{ startIndex: 11, type: 'white.pq' },
+			{ startIndex: 12, type: 'keyword.pq' },
+			{ startIndex: 16, type: 'white.pq' },
+			{ startIndex: 17, type: 'identifier.pq' }
+		]
+	}],
+
+	[{
+		line: '  #table({})',
+		tokens: [
+			{ startIndex: 0, type: 'white.pq' },
+			{ startIndex: 2, type: 'constructor.pq' },
+			{ startIndex: 8, type: 'delimiter.parenthesis.pq' },
+			{ startIndex: 9, type: 'delimiter.brackets.pq' },
+			{ startIndex: 11, type: "delimiter.parenthesis.pq" }
+		]
+	}],
+
+	[{
+		line: 'param as number',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 5, type: 'white.pq' },
+			{ startIndex: 6, type: 'keyword.pq' },
+			{ startIndex: 8, type: 'white.pq' },
+			{ startIndex: 9, type: 'type.pq' }
+		]
+	}],
+
+	[{
+		line: 'type table',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.pq' },
+			{ startIndex: 4, type: 'white.pq' },
+			{ startIndex: 5, type: 'type.pq' }
+		]
+	}],
+
+	[{
+		line: 'if (a = #nan) then null else a',
+		tokens: [
+			{ startIndex: 0, type: "keyword.pq" },
+			{ startIndex: 2, type: "white.pq" },
+			{ startIndex: 3, type: "delimiter.parenthesis.pq"},
+			{ startIndex: 4, type: "identifier.pq" },
+			{ startIndex: 5, type: "white.pq" },
+			{ startIndex: 6, type: "operators.pq" },
+			{ startIndex: 7, type: "white.pq" },
+			{ startIndex: 8, type: "constant.pq" },
+			{ startIndex: 12, type: "delimiter.parenthesis.pq"},
+			{ startIndex: 13, type: "white.pq" },
+			{ startIndex: 14, type: "keyword.pq" },
+			{ startIndex: 18, type: "white.pq" },
+			{ startIndex: 19, type: "type.pq" },
+			{ startIndex: 23, type: "white.pq" },
+			{ startIndex: 24, type: "keyword.pq" },
+			{ startIndex: 28, type: "white.pq" },
+			{ startIndex: 29, type: "identifier.pq" },
+		]
+	}],
+
+	// built-ins
+	[{
+		line: 'Text.From(1)',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.function.pq' },
+			{ startIndex: 9, type: 'delimiter.parenthesis.pq'},
+			{ startIndex: 10, type: 'number.pq' },
+			{ startIndex: 11, type: 'delimiter.parenthesis.pq'}
+		]
+	}],
+
+	[{
+		line: 'Text.ToBinary("123", BinaryEncoding.Base64)',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.function.pq' },
+			{ startIndex: 13, type: 'delimiter.parenthesis.pq' },
+			{ startIndex: 14, type: 'string.pq' },
+			{ startIndex: 19, type: 'delimiter.pq' },
+			{ startIndex: 20, type: 'white.pq' },
+			{ startIndex: 21, type: 'constant.pq' },
+			{ startIndex: 42, type: 'delimiter.parenthesis.pq' }
+		]
+	}],
+
+	[{
+		line: 'Int8.Type',
+		tokens: [
+			{ startIndex: 0, type: 'type.pq' }
+		]
+	}],
+
+	[{
+		line: 'DB2.Database',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.function.pq' }
+		]
+	}],
+
+	[{
+		line: 'RelativePosition.Type',
+		tokens: [
+			{ startIndex: 0, type: 'type.pq' }
+		]
+	}],
+
+	// other statements
+	[{
+		line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;',
+		tokens: [
+			{ startIndex: 0, type: "delimiter.square.pq" },
+			{ startIndex: 1, type: "identifier.pq" },
+			{ startIndex: 8, type: "operators.pq" },
+			{ startIndex: 9, type: "string.pq" },
+			{ startIndex: 18, type: "delimiter.square.pq" },
+			{ startIndex: 19, type: "white.pq" },
+			{ startIndex: 20, type: "keyword.pq" },
+			{ startIndex: 27, type: "white.pq" },
+			{ startIndex: 28, type: "identifier.pq" },
+			{ startIndex: 31, type: "delimiter.pq" },
+			{ startIndex: 32, type: "white.pq" },
+			{ startIndex: 33, type: "keyword.pq" },
+			{ startIndex: 39, type: "white.pq" },
+			{ startIndex: 40, type: "identifier.pq" },
+			{ startIndex: 51, type: "white.pq" },
+			{ startIndex: 52, type: "operators.pq" },
+			{ startIndex: 53, type: "white.pq" },
+			{ startIndex: 54, type: "number.pq" },
+			{ startIndex: 55, type: "delimiter.pq" }
+		]
+	}],
+
+	[{
+		line: 'isFunctionthen = 1;// comment',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.pq' },
+			{ startIndex: 14, type: 'white.pq' },
+			{ startIndex: 15, type: 'operators.pq' },
+			{ startIndex: 16, type: 'white.pq' },
+			{ startIndex: 17, type: 'number.pq' },
+			{ startIndex: 18, type: 'delimiter.pq' },
+			{ startIndex: 19, type: 'comment.pq' },
+		]
+	}],
+
+	[{
+		line: '@RecursiveFunction()+@Rec.Func()',
+		tokens: [
+			{ startIndex: 0, type: 'operators.pq' },
+			{ startIndex: 1, type: 'identifier.pq' },
+			{ startIndex: 18, type: "delimiter.parenthesis.pq"},
+			{ startIndex: 20, type: 'operators.pq' },
+			{ startIndex: 22, type: 'identifier.pq' },
+			{ startIndex: 30, type: "delimiter.parenthesis.pq"},
+		]
+	}],
+]);

src/powerquery/powerquery.ts

@@ -0,0 +1,934 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['[', ']'], ['(', ')'], ['{', '}']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment', 'identifier'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment', 'identifier'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment', 'identifier'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment', 'identifier'] },
+	]
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.pq',
+	ignoreCase: false,
+
+	brackets: [
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '{', close: '}', token: 'delimiter.brackets' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	operatorKeywords: [
+		"and",
+		"not",
+		"or"
+	],
+
+	keywords: [
+		"as",
+		"each",
+		"else",
+		"error",
+		"false",
+		"if",
+		"in",
+		"is",
+		"let",
+		"meta",
+		"otherwise",
+		"section",
+		"shared",
+		"then",
+		"true",
+		"try",
+		"type"
+	],
+
+	constructors: [
+		"#binary",
+		"#date",
+		"#datetime",
+		"#datetimezone",
+		"#duration",
+		"#table",
+		"#time"
+	],
+
+	constants: [
+		"#infinity",
+		"#nan",
+		"#sections",
+		"#shared"
+	],
+
+	typeKeywords: [
+		"action",
+		"any",
+		"anynonnull",
+		"none",
+		"null",
+		"logical",
+		"number",
+		"time",
+		"date",
+		"datetime",
+		"datetimezone",
+		"duration",
+		"text",
+		"binary",
+		"list",
+		"record",
+		"table",
+		"function"
+	],
+
+	builtinFunctions: [
+		"Access.Database",
+		"Action.Return",
+		"Action.Sequence",
+		"Action.Try",
+		"ActiveDirectory.Domains",
+		"AdoDotNet.DataSource",
+		"AdoDotNet.Query",
+		"AdobeAnalytics.Cubes",
+		"AnalysisServices.Database",
+		"AnalysisServices.Databases",
+		"AzureStorage.BlobContents",
+		"AzureStorage.Blobs",
+		"AzureStorage.Tables",
+		"Binary.Buffer",
+		"Binary.Combine",
+		"Binary.Compress",
+		"Binary.Decompress",
+		"Binary.End",
+		"Binary.From",
+		"Binary.FromList",
+		"Binary.FromText",
+		"Binary.InferContentType",
+		"Binary.Length",
+		"Binary.ToList",
+		"Binary.ToText",
+		"BinaryFormat.7BitEncodedSignedInteger",
+		"BinaryFormat.7BitEncodedUnsignedInteger",
+		"BinaryFormat.Binary",
+		"BinaryFormat.Byte",
+		"BinaryFormat.ByteOrder",
+		"BinaryFormat.Choice",
+		"BinaryFormat.Decimal",
+		"BinaryFormat.Double",
+		"BinaryFormat.Group",
+		"BinaryFormat.Length",
+		"BinaryFormat.List",
+		"BinaryFormat.Null",
+		"BinaryFormat.Record",
+		"BinaryFormat.SignedInteger16",
+		"BinaryFormat.SignedInteger32",
+		"BinaryFormat.SignedInteger64",
+		"BinaryFormat.Single",
+		"BinaryFormat.Text",
+		"BinaryFormat.Transform",
+		"BinaryFormat.UnsignedInteger16",
+		"BinaryFormat.UnsignedInteger32",
+		"BinaryFormat.UnsignedInteger64",
+		"Byte.From",
+		"Character.FromNumber",
+		"Character.ToNumber",
+		"Combiner.CombineTextByDelimiter",
+		"Combiner.CombineTextByEachDelimiter",
+		"Combiner.CombineTextByLengths",
+		"Combiner.CombineTextByPositions",
+		"Combiner.CombineTextByRanges",
+		"Comparer.Equals",
+		"Comparer.FromCulture",
+		"Comparer.Ordinal",
+		"Comparer.OrdinalIgnoreCase",
+		"Csv.Document",
+		"Cube.AddAndExpandDimensionColumn",
+		"Cube.AddMeasureColumn",
+		"Cube.ApplyParameter",
+		"Cube.AttributeMemberId",
+		"Cube.AttributeMemberProperty",
+		"Cube.CollapseAndRemoveColumns",
+		"Cube.Dimensions",
+		"Cube.DisplayFolders",
+		"Cube.Measures",
+		"Cube.Parameters",
+		"Cube.Properties",
+		"Cube.PropertyKey",
+		"Cube.ReplaceDimensions",
+		"Cube.Transform",
+		"Currency.From",
+		"DB2.Database",
+		"Date.AddDays",
+		"Date.AddMonths",
+		"Date.AddQuarters",
+		"Date.AddWeeks",
+		"Date.AddYears",
+		"Date.Day",
+		"Date.DayOfWeek",
+		"Date.DayOfWeekName",
+		"Date.DayOfYear",
+		"Date.DaysInMonth",
+		"Date.EndOfDay",
+		"Date.EndOfMonth",
+		"Date.EndOfQuarter",
+		"Date.EndOfWeek",
+		"Date.EndOfYear",
+		"Date.From",
+		"Date.FromText",
+		"Date.IsInCurrentDay",
+		"Date.IsInCurrentMonth",
+		"Date.IsInCurrentQuarter",
+		"Date.IsInCurrentWeek",
+		"Date.IsInCurrentYear",
+		"Date.IsInNextDay",
+		"Date.IsInNextMonth",
+		"Date.IsInNextNDays",
+		"Date.IsInNextNMonths",
+		"Date.IsInNextNQuarters",
+		"Date.IsInNextNWeeks",
+		"Date.IsInNextNYears",
+		"Date.IsInNextQuarter",
+		"Date.IsInNextWeek",
+		"Date.IsInNextYear",
+		"Date.IsInPreviousDay",
+		"Date.IsInPreviousMonth",
+		"Date.IsInPreviousNDays",
+		"Date.IsInPreviousNMonths",
+		"Date.IsInPreviousNQuarters",
+		"Date.IsInPreviousNWeeks",
+		"Date.IsInPreviousNYears",
+		"Date.IsInPreviousQuarter",
+		"Date.IsInPreviousWeek",
+		"Date.IsInPreviousYear",
+		"Date.IsInYearToDate",
+		"Date.IsLeapYear",
+		"Date.Month",
+		"Date.MonthName",
+		"Date.QuarterOfYear",
+		"Date.StartOfDay",
+		"Date.StartOfMonth",
+		"Date.StartOfQuarter",
+		"Date.StartOfWeek",
+		"Date.StartOfYear",
+		"Date.ToRecord",
+		"Date.ToText",
+		"Date.WeekOfMonth",
+		"Date.WeekOfYear",
+		"Date.Year",
+		"DateTime.AddZone",
+		"DateTime.Date",
+		"DateTime.FixedLocalNow",
+		"DateTime.From",
+		"DateTime.FromFileTime",
+		"DateTime.FromText",
+		"DateTime.IsInCurrentHour",
+		"DateTime.IsInCurrentMinute",
+		"DateTime.IsInCurrentSecond",
+		"DateTime.IsInNextHour",
+		"DateTime.IsInNextMinute",
+		"DateTime.IsInNextNHours",
+		"DateTime.IsInNextNMinutes",
+		"DateTime.IsInNextNSeconds",
+		"DateTime.IsInNextSecond",
+		"DateTime.IsInPreviousHour",
+		"DateTime.IsInPreviousMinute",
+		"DateTime.IsInPreviousNHours",
+		"DateTime.IsInPreviousNMinutes",
+		"DateTime.IsInPreviousNSeconds",
+		"DateTime.IsInPreviousSecond",
+		"DateTime.LocalNow",
+		"DateTime.Time",
+		"DateTime.ToRecord",
+		"DateTime.ToText",
+		"DateTimeZone.FixedLocalNow",
+		"DateTimeZone.FixedUtcNow",
+		"DateTimeZone.From",
+		"DateTimeZone.FromFileTime",
+		"DateTimeZone.FromText",
+		"DateTimeZone.LocalNow",
+		"DateTimeZone.RemoveZone",
+		"DateTimeZone.SwitchZone",
+		"DateTimeZone.ToLocal",
+		"DateTimeZone.ToRecord",
+		"DateTimeZone.ToText",
+		"DateTimeZone.ToUtc",
+		"DateTimeZone.UtcNow",
+		"DateTimeZone.ZoneHours",
+		"DateTimeZone.ZoneMinutes",
+		"Decimal.From",
+		"Diagnostics.ActivityId",
+		"Diagnostics.Trace",
+		"DirectQueryCapabilities.From",
+		"Double.From",
+		"Duration.Days",
+		"Duration.From",
+		"Duration.FromText",
+		"Duration.Hours",
+		"Duration.Minutes",
+		"Duration.Seconds",
+		"Duration.ToRecord",
+		"Duration.ToText",
+		"Duration.TotalDays",
+		"Duration.TotalHours",
+		"Duration.TotalMinutes",
+		"Duration.TotalSeconds",
+		"Embedded.Value",
+		"Error.Record",
+		"Excel.CurrentWorkbook",
+		"Excel.Workbook",
+		"Exchange.Contents",
+		"Expression.Constant",
+		"Expression.Evaluate",
+		"Expression.Identifier",
+		"Facebook.Graph",
+		"File.Contents",
+		"Folder.Contents",
+		"Folder.Files",
+		"Function.From",
+		"Function.Invoke",
+		"Function.InvokeAfter",
+		"Function.IsDataSource",
+		"GoogleAnalytics.Accounts",
+		"Guid.From",
+		"HdInsight.Containers",
+		"HdInsight.Contents",
+		"HdInsight.Files",
+		"Hdfs.Contents",
+		"Hdfs.Files",
+		"Informix.Database",
+		"Int16.From",
+		"Int32.From",
+		"Int64.From",
+		"Int8.From",
+		"ItemExpression.From",
+		"Json.Document",
+		"Json.FromValue",
+		"Lines.FromBinary",
+		"Lines.FromText",
+		"Lines.ToBinary",
+		"Lines.ToText",
+		"List.Accumulate",
+		"List.AllTrue",
+		"List.Alternate",
+		"List.AnyTrue",
+		"List.Average",
+		"List.Buffer",
+		"List.Combine",
+		"List.Contains",
+		"List.ContainsAll",
+		"List.ContainsAny",
+		"List.Count",
+		"List.Covariance",
+		"List.DateTimeZones",
+		"List.DateTimes",
+		"List.Dates",
+		"List.Difference",
+		"List.Distinct",
+		"List.Durations",
+		"List.FindText",
+		"List.First",
+		"List.FirstN",
+		"List.Generate",
+		"List.InsertRange",
+		"List.Intersect",
+		"List.IsDistinct",
+		"List.IsEmpty",
+		"List.Last",
+		"List.LastN",
+		"List.MatchesAll",
+		"List.MatchesAny",
+		"List.Max",
+		"List.MaxN",
+		"List.Median",
+		"List.Min",
+		"List.MinN",
+		"List.Mode",
+		"List.Modes",
+		"List.NonNullCount",
+		"List.Numbers",
+		"List.PositionOf",
+		"List.PositionOfAny",
+		"List.Positions",
+		"List.Product",
+		"List.Random",
+		"List.Range",
+		"List.RemoveFirstN",
+		"List.RemoveItems",
+		"List.RemoveLastN",
+		"List.RemoveMatchingItems",
+		"List.RemoveNulls",
+		"List.RemoveRange",
+		"List.Repeat",
+		"List.ReplaceMatchingItems",
+		"List.ReplaceRange",
+		"List.ReplaceValue",
+		"List.Reverse",
+		"List.Select",
+		"List.Single",
+		"List.SingleOrDefault",
+		"List.Skip",
+		"List.Sort",
+		"List.StandardDeviation",
+		"List.Sum",
+		"List.Times",
+		"List.Transform",
+		"List.TransformMany",
+		"List.Union",
+		"List.Zip",
+		"Logical.From",
+		"Logical.FromText",
+		"Logical.ToText",
+		"MQ.Queue",
+		"MySQL.Database",
+		"Number.Abs",
+		"Number.Acos",
+		"Number.Asin",
+		"Number.Atan",
+		"Number.Atan2",
+		"Number.BitwiseAnd",
+		"Number.BitwiseNot",
+		"Number.BitwiseOr",
+		"Number.BitwiseShiftLeft",
+		"Number.BitwiseShiftRight",
+		"Number.BitwiseXor",
+		"Number.Combinations",
+		"Number.Cos",
+		"Number.Cosh",
+		"Number.Exp",
+		"Number.Factorial",
+		"Number.From",
+		"Number.FromText",
+		"Number.IntegerDivide",
+		"Number.IsEven",
+		"Number.IsNaN",
+		"Number.IsOdd",
+		"Number.Ln",
+		"Number.Log",
+		"Number.Log10",
+		"Number.Mod",
+		"Number.Permutations",
+		"Number.Power",
+		"Number.Random",
+		"Number.RandomBetween",
+		"Number.Round",
+		"Number.RoundAwayFromZero",
+		"Number.RoundDown",
+		"Number.RoundTowardZero",
+		"Number.RoundUp",
+		"Number.Sign",
+		"Number.Sin",
+		"Number.Sinh",
+		"Number.Sqrt",
+		"Number.Tan",
+		"Number.Tanh",
+		"Number.ToText",
+		"OData.Feed",
+		"Odbc.DataSource",
+		"Odbc.Query",
+		"OleDb.DataSource",
+		"OleDb.Query",
+		"Oracle.Database",
+		"Percentage.From",
+		"PostgreSQL.Database",
+		"RData.FromBinary",
+		"Record.AddField",
+		"Record.Combine",
+		"Record.Field",
+		"Record.FieldCount",
+		"Record.FieldNames",
+		"Record.FieldOrDefault",
+		"Record.FieldValues",
+		"Record.FromList",
+		"Record.FromTable",
+		"Record.HasFields",
+		"Record.RemoveFields",
+		"Record.RenameFields",
+		"Record.ReorderFields",
+		"Record.SelectFields",
+		"Record.ToList",
+		"Record.ToTable",
+		"Record.TransformFields",
+		"Replacer.ReplaceText",
+		"Replacer.ReplaceValue",
+		"RowExpression.Column",
+		"RowExpression.From",
+		"Salesforce.Data",
+		"Salesforce.Reports",
+		"SapBusinessWarehouse.Cubes",
+		"SapHana.Database",
+		"SharePoint.Contents",
+		"SharePoint.Files",
+		"SharePoint.Tables",
+		"Single.From",
+		"Soda.Feed",
+		"Splitter.SplitByNothing",
+		"Splitter.SplitTextByAnyDelimiter",
+		"Splitter.SplitTextByDelimiter",
+		"Splitter.SplitTextByEachDelimiter",
+		"Splitter.SplitTextByLengths",
+		"Splitter.SplitTextByPositions",
+		"Splitter.SplitTextByRanges",
+		"Splitter.SplitTextByRepeatedLengths",
+		"Splitter.SplitTextByWhitespace",
+		"Sql.Database",
+		"Sql.Databases",
+		"SqlExpression.SchemaFrom",
+		"SqlExpression.ToExpression",
+		"Sybase.Database",
+		"Table.AddColumn",
+		"Table.AddIndexColumn",
+		"Table.AddJoinColumn",
+		"Table.AddKey",
+		"Table.AggregateTableColumn",
+		"Table.AlternateRows",
+		"Table.Buffer",
+		"Table.Column",
+		"Table.ColumnCount",
+		"Table.ColumnNames",
+		"Table.ColumnsOfType",
+		"Table.Combine",
+		"Table.CombineColumns",
+		"Table.Contains",
+		"Table.ContainsAll",
+		"Table.ContainsAny",
+		"Table.DemoteHeaders",
+		"Table.Distinct",
+		"Table.DuplicateColumn",
+		"Table.ExpandListColumn",
+		"Table.ExpandRecordColumn",
+		"Table.ExpandTableColumn",
+		"Table.FillDown",
+		"Table.FillUp",
+		"Table.FilterWithDataTable",
+		"Table.FindText",
+		"Table.First",
+		"Table.FirstN",
+		"Table.FirstValue",
+		"Table.FromColumns",
+		"Table.FromList",
+		"Table.FromPartitions",
+		"Table.FromRecords",
+		"Table.FromRows",
+		"Table.FromValue",
+		"Table.Group",
+		"Table.HasColumns",
+		"Table.InsertRows",
+		"Table.IsDistinct",
+		"Table.IsEmpty",
+		"Table.Join",
+		"Table.Keys",
+		"Table.Last",
+		"Table.LastN",
+		"Table.MatchesAllRows",
+		"Table.MatchesAnyRows",
+		"Table.Max",
+		"Table.MaxN",
+		"Table.Min",
+		"Table.MinN",
+		"Table.NestedJoin",
+		"Table.Partition",
+		"Table.PartitionValues",
+		"Table.Pivot",
+		"Table.PositionOf",
+		"Table.PositionOfAny",
+		"Table.PrefixColumns",
+		"Table.Profile",
+		"Table.PromoteHeaders",
+		"Table.Range",
+		"Table.RemoveColumns",
+		"Table.RemoveFirstN",
+		"Table.RemoveLastN",
+		"Table.RemoveMatchingRows",
+		"Table.RemoveRows",
+		"Table.RemoveRowsWithErrors",
+		"Table.RenameColumns",
+		"Table.ReorderColumns",
+		"Table.Repeat",
+		"Table.ReplaceErrorValues",
+		"Table.ReplaceKeys",
+		"Table.ReplaceMatchingRows",
+		"Table.ReplaceRelationshipIdentity",
+		"Table.ReplaceRows",
+		"Table.ReplaceValue",
+		"Table.ReverseRows",
+		"Table.RowCount",
+		"Table.Schema",
+		"Table.SelectColumns",
+		"Table.SelectRows",
+		"Table.SelectRowsWithErrors",
+		"Table.SingleRow",
+		"Table.Skip",
+		"Table.Sort",
+		"Table.SplitColumn",
+		"Table.ToColumns",
+		"Table.ToList",
+		"Table.ToRecords",
+		"Table.ToRows",
+		"Table.TransformColumnNames",
+		"Table.TransformColumnTypes",
+		"Table.TransformColumns",
+		"Table.TransformRows",
+		"Table.Transpose",
+		"Table.Unpivot",
+		"Table.UnpivotOtherColumns",
+		"Table.View",
+		"Table.ViewFunction",
+		"TableAction.DeleteRows",
+		"TableAction.InsertRows",
+		"TableAction.UpdateRows",
+		"Tables.GetRelationships",
+		"Teradata.Database",
+		"Text.AfterDelimiter",
+		"Text.At",
+		"Text.BeforeDelimiter",
+		"Text.BetweenDelimiters",
+		"Text.Clean",
+		"Text.Combine",
+		"Text.Contains",
+		"Text.End",
+		"Text.EndsWith",
+		"Text.Format",
+		"Text.From",
+		"Text.FromBinary",
+		"Text.Insert",
+		"Text.Length",
+		"Text.Lower",
+		"Text.Middle",
+		"Text.NewGuid",
+		"Text.PadEnd",
+		"Text.PadStart",
+		"Text.PositionOf",
+		"Text.PositionOfAny",
+		"Text.Proper",
+		"Text.Range",
+		"Text.Remove",
+		"Text.RemoveRange",
+		"Text.Repeat",
+		"Text.Replace",
+		"Text.ReplaceRange",
+		"Text.Select",
+		"Text.Split",
+		"Text.SplitAny",
+		"Text.Start",
+		"Text.StartsWith",
+		"Text.ToBinary",
+		"Text.ToList",
+		"Text.Trim",
+		"Text.TrimEnd",
+		"Text.TrimStart",
+		"Text.Upper",
+		"Time.EndOfHour",
+		"Time.From",
+		"Time.FromText",
+		"Time.Hour",
+		"Time.Minute",
+		"Time.Second",
+		"Time.StartOfHour",
+		"Time.ToRecord",
+		"Time.ToText",
+		"Type.AddTableKey",
+		"Type.ClosedRecord",
+		"Type.Facets",
+		"Type.ForFunction",
+		"Type.ForRecord",
+		"Type.FunctionParameters",
+		"Type.FunctionRequiredParameters",
+		"Type.FunctionReturn",
+		"Type.Is",
+		"Type.IsNullable",
+		"Type.IsOpenRecord",
+		"Type.ListItem",
+		"Type.NonNullable",
+		"Type.OpenRecord",
+		"Type.RecordFields",
+		"Type.ReplaceFacets",
+		"Type.ReplaceTableKeys",
+		"Type.TableColumn",
+		"Type.TableKeys",
+		"Type.TableRow",
+		"Type.TableSchema",
+		"Type.Union",
+		"Uri.BuildQueryString",
+		"Uri.Combine",
+		"Uri.EscapeDataString",
+		"Uri.Parts",
+		"Value.Add",
+		"Value.As",
+		"Value.Compare",
+		"Value.Divide",
+		"Value.Equals",
+		"Value.Firewall",
+		"Value.FromText",
+		"Value.Is",
+		"Value.Metadata",
+		"Value.Multiply",
+		"Value.NativeQuery",
+		"Value.NullableEquals",
+		"Value.RemoveMetadata",
+		"Value.ReplaceMetadata",
+		"Value.ReplaceType",
+		"Value.Subtract",
+		"Value.Type",
+		"ValueAction.NativeStatement",
+		"ValueAction.Replace",
+		"Variable.Value",
+		"Web.Contents",
+		"Web.Page",
+		"WebAction.Request",
+		"Xml.Document",
+		"Xml.Tables"
+	],
+
+	builtinConstants: [
+		"BinaryEncoding.Base64",
+		"BinaryEncoding.Hex",
+		"BinaryOccurrence.Optional",
+		"BinaryOccurrence.Repeating",
+		"BinaryOccurrence.Required",
+		"ByteOrder.BigEndian",
+		"ByteOrder.LittleEndian",
+		"Compression.Deflate",
+		"Compression.GZip",
+		"CsvStyle.QuoteAfterDelimiter",
+		"CsvStyle.QuoteAlways",
+		"Culture.Current",
+		"Day.Friday",
+		"Day.Monday",
+		"Day.Saturday",
+		"Day.Sunday",
+		"Day.Thursday",
+		"Day.Tuesday",
+		"Day.Wednesday",
+		"ExtraValues.Error",
+		"ExtraValues.Ignore",
+		"ExtraValues.List",
+		"GroupKind.Global",
+		"GroupKind.Local",
+		"JoinAlgorithm.Dynamic",
+		"JoinAlgorithm.LeftHash",
+		"JoinAlgorithm.LeftIndex",
+		"JoinAlgorithm.PairwiseHash",
+		"JoinAlgorithm.RightHash",
+		"JoinAlgorithm.RightIndex",
+		"JoinAlgorithm.SortMerge",
+		"JoinKind.FullOuter",
+		"JoinKind.Inner",
+		"JoinKind.LeftAnti",
+		"JoinKind.LeftOuter",
+		"JoinKind.RightAnti",
+		"JoinKind.RightOuter",
+		"JoinSide.Left",
+		"JoinSide.Right",
+		"MissingField.Error",
+		"MissingField.Ignore",
+		"MissingField.UseNull",
+		"Number.E",
+		"Number.Epsilon",
+		"Number.NaN",
+		"Number.NegativeInfinity",
+		"Number.PI",
+		"Number.PositiveInfinity",
+		"Occurrence.All",
+		"Occurrence.First",
+		"Occurrence.Last",
+		"Occurrence.Optional",
+		"Occurrence.Repeating",
+		"Occurrence.Required",
+		"Order.Ascending",
+		"Order.Descending",
+		"Precision.Decimal",
+		"Precision.Double",
+		"QuoteStyle.Csv",
+		"QuoteStyle.None",
+		"RelativePosition.FromEnd",
+		"RelativePosition.FromStart",
+		"RoundingMode.AwayFromZero",
+		"RoundingMode.Down",
+		"RoundingMode.ToEven",
+		"RoundingMode.TowardZero",
+		"RoundingMode.Up",
+		"SapHanaDistribution.All",
+		"SapHanaDistribution.Connection",
+		"SapHanaDistribution.Off",
+		"SapHanaDistribution.Statement",
+		"SapHanaRangeOperator.Equals",
+		"SapHanaRangeOperator.GreaterThan",
+		"SapHanaRangeOperator.GreaterThanOrEquals",
+		"SapHanaRangeOperator.LessThan",
+		"SapHanaRangeOperator.LessThanOrEquals",
+		"SapHanaRangeOperator.NotEquals",
+		"TextEncoding.Ascii",
+		"TextEncoding.BigEndianUnicode",
+		"TextEncoding.Unicode",
+		"TextEncoding.Utf16",
+		"TextEncoding.Utf8",
+		"TextEncoding.Windows",
+		"TraceLevel.Critical",
+		"TraceLevel.Error",
+		"TraceLevel.Information",
+		"TraceLevel.Verbose",
+		"TraceLevel.Warning",
+		"WebMethod.Delete",
+		"WebMethod.Get",
+		"WebMethod.Head",
+		"WebMethod.Patch",
+		"WebMethod.Post",
+		"WebMethod.Put"
+	],
+
+	builtinTypes: [
+		"Action.Type",
+		"Any.Type",
+		"Binary.Type",
+		"BinaryEncoding.Type",
+		"BinaryOccurrence.Type",
+		"Byte.Type",
+		"ByteOrder.Type",
+		"Character.Type",
+		"Compression.Type",
+		"CsvStyle.Type",
+		"Currency.Type",
+		"Date.Type",
+		"DateTime.Type",
+		"DateTimeZone.Type",
+		"Day.Type",
+		"Decimal.Type",
+		"Double.Type",
+		"Duration.Type",
+		"ExtraValues.Type",
+		"Function.Type",
+		"GroupKind.Type",
+		"Guid.Type",
+		"Int16.Type",
+		"Int32.Type",
+		"Int64.Type",
+		"Int8.Type",
+		"JoinAlgorithm.Type",
+		"JoinKind.Type",
+		"JoinSide.Type",
+		"List.Type",
+		"Logical.Type",
+		"MissingField.Type",
+		"None.Type",
+		"Null.Type",
+		"Number.Type",
+		"Occurrence.Type",
+		"Order.Type",
+		"Password.Type",
+		"Percentage.Type",
+		"Precision.Type",
+		"QuoteStyle.Type",
+		"Record.Type",
+		"RelativePosition.Type",
+		"RoundingMode.Type",
+		"SapHanaDistribution.Type",
+		"SapHanaRangeOperator.Type",
+		"Single.Type",
+		"Table.Type",
+		"Text.Type",
+		"TextEncoding.Type",
+		"Time.Type",
+		"TraceLevel.Type",
+		"Type.Type",
+		"Uri.Type",
+		"WebMethod.Type"
+	],
+
+	tokenizer: {
+		root: [
+			// quoted identifier
+			[/#"[\w \.]+"/, "identifier.quote"],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, "number.float"],
+			[/0[xX][0-9a-fA-F]+/, "number.hex"],
+			[/\d+([eE][\-+]?\d+)?/, "number"],
+
+			// keywords
+			[/(#?[a-z]+)\b/,
+				{
+					cases: {
+						"@typeKeywords": "type",
+						"@keywords": "keyword",
+						"@constants": "constant",
+						"@constructors": "constructor",
+						"@operatorKeywords": "operators",
+						"@default": "identifier"
+					}
+				}
+			],
+
+			// built-in types
+			[/\b([A-Z][a-zA-Z0-9]+\.Type)\b/,
+				{
+					cases: {
+						"@builtinTypes": "type",
+						"@default": "identifier"
+					}
+				}
+			],
+
+			// other built-ins
+			[/\b([A-Z][a-zA-Z0-9]+\.[A-Z][a-zA-Z0-9]+)\b/,
+				{
+					cases: {
+						"@builtinFunctions": "keyword.function",
+						"@builtinConstants": "constant",
+						"@default": "identifier"
+					}
+				}
+			],
+
+			// other identifiers
+			[/\b([a-zA-Z_][\w\.]*)\b/, "identifier"],
+
+			{ include: "@whitespace" },
+			{ include: "@comments" },
+			{ include: "@strings" },
+
+			[/[{}()\[\]]/, "@brackets"],
+			[/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, "operators"],
+			[/[,;]/, "delimiter"],
+		],
+
+		whitespace: [
+			[/\s+/, "white"]
+		],
+
+		comments: [
+			["\\/\\*", "comment", "@comment"],
+			["\\/\\/+.*", "comment"]
+		],
+
+		comment: [
+			["\\*\\/", "comment", "@pop"],
+			[".", "comment"]
+		],
+
+		strings: [
+			["\"", "string", "@string"]
+		],
+
+		string: [
+			["\"\"", "string.escape"],
+			["\"", "string", "@pop"],
+			[".", "string"]
+		]
+	}
+};

src/powershell/powershell.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'powershell',
 	extensions: ['.ps1', '.psm1', '.psd1'],
 	aliases: ['PowerShell', 'powershell', 'ps', 'ps1'],
-	loader: () => _monaco.Promise.wrap(import('./powershell'))
+	loader: () => import('./powershell')
 });

src/pug/pug.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'pug',
 	extensions: ['.jade', '.pug'],
 	aliases: ['Pug', 'Jade', 'jade'],
-	loader: () => _monaco.Promise.wrap(import('./pug'))
+	loader: () => import('./pug')
 });

src/python/python.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'python',
 	extensions: ['.py', '.rpy', '.pyw', '.cpy', '.gyp', '.gypi'],
 	aliases: ['Python', 'py'],
 	firstLine: '^#!/.*\\bpython[0-9.-]*\\b',
-	loader: () => _monaco.Promise.wrap(import('./python'))
+	loader: () => import('./python')
 });

src/python/python.test.ts

@@ -92,6 +92,69 @@ testTokenization('python', [
 		]
 	}],
 
+	// https://github.com/Microsoft/monaco-editor/issues/1170
+	[{
+		line: 'def f():',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.python' },
+			{ startIndex: 3, type: 'white.python' },
+			{ startIndex: 4, type: 'identifier.python' },
+			{ startIndex: 5, type: 'delimiter.parenthesis.python' },
+			{ startIndex: 7, type: 'delimiter.python' },
+		]
+	}, {
+		line: '   """multi',
+		tokens: [
+			{ startIndex: 0, type: 'white.python' },
+			{ startIndex: 3, type: 'string.python' },
+		]
+	}, {
+		line: '   line',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+		]
+	}, {
+		line: '   comment',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+		]
+	}, {
+		line: '   """ + """',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+			{ startIndex: 6, type: 'white.python' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'white.python' },
+			{ startIndex: 9, type: 'string.python' },
+		]
+	}, {
+		line: '   another',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+		]
+	}, {
+		line: '   multi',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+		]
+	}, {
+		line: '   line',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+		]
+	}, {
+		line: '   comment"""',
+		tokens: [
+			{ startIndex: 0, type: 'string.python' },
+		]
+	}, {
+		line: '   code',
+		tokens: [
+			{ startIndex: 0, type: 'white.python' },
+			{ startIndex: 3, type: 'identifier.python' },
+		]
+	}],
+
 	// Numbers
 	[{
 		line: '0xAcBFd',

src/python/python.ts

@@ -8,6 +8,9 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
+// Allow for running under nodejs/requirejs in tests
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
 export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '#',
@@ -32,6 +35,12 @@ export const conf: IRichLanguageConfiguration = {
 		{ open: '"', close: '"' },
 		{ open: '\'', close: '\'' },
 	],
+	onEnterRules: [
+		{
+			beforeText: new RegExp("^\\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async).*?:\\s*$"),
+			action: { indentAction: _monaco.languages.IndentAction.Indent }
+		}
+	],
 	folding: {
 		offSide: true,
 		markers: {
@@ -206,19 +215,20 @@ export const language = <ILanguage>{
 		whitespace: [
 			[/\s+/, 'white'],
 			[/(^#.*$)/, 'comment'],
-			[/('''.*''')|(""".*""")/, 'string'],
-			[/'''.*$/, 'string', '@endDocString'],
-			[/""".*$/, 'string', '@endDblDocString']
+			[/'''/, 'string', '@endDocString'],
+			[/"""/, 'string', '@endDblDocString']
 		],
 		endDocString: [
+			[/[^']+/, 'string'],
 			[/\\'/, 'string'],
-			[/.*'''/, 'string', '@popall'],
-			[/.*$/, 'string']
+			[/'''/, 'string', '@popall'],
+			[/'/, 'string']
 		],
 		endDblDocString: [
+			[/[^"]+/, 'string'],
 			[/\\"/, 'string'],
-			[/.*"""/, 'string', '@popall'],
-			[/.*$/, 'string']
+			[/"""/, 'string', '@popall'],
+			[/"/, 'string']
 		],
 
 		// Recognize hex, negatives, decimals, imaginaries, longs, and scientific notation
@@ -235,18 +245,18 @@ export const language = <ILanguage>{
 			[/"/, 'string.escape', '@dblStringBody']
 		],
 		stringBody: [
+			[/[^\\']+$/, 'string', '@popall'],
+			[/[^\\']+/, 'string'],
 			[/\\./, 'string'],
 			[/'/, 'string.escape', '@popall'],
-			[/.(?=.*')/, 'string'],
-			[/.*\\$/, 'string'],
-			[/.*$/, 'string', '@popall']
+			[/\\$/, 'string']
 		],
 		dblStringBody: [
+			[/[^\\"]+$/, 'string', '@popall'],
+			[/[^\\"]+/, 'string'],
 			[/\\./, 'string'],
 			[/"/, 'string.escape', '@popall'],
-			[/.(?=.*")/, 'string'],
-			[/.*\\$/, 'string'],
-			[/.*$/, 'string', '@popall']
+			[/\\$/, 'string']
 		]
 	}
 };

src/r/r.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'r',
 	extensions: ['.r', '.rhistory', '.rprofile', '.rt'],
 	aliases: ['R', 'r'],
-	loader: () => _monaco.Promise.wrap(import('./r'))
+	loader: () => import('./r')
 });

src/razor/razor.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'razor',
 	extensions: ['.cshtml'],
 	aliases: ['Razor', 'razor'],
 	mimetypes: ['text/x-cshtml'],
-	loader: () => _monaco.Promise.wrap(import('./razor'))
+	loader: () => import('./razor')
 });

src/redis/redis.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'redis',
 	extensions: ['.redis'],
 	aliases: ['redis'],
-	loader: () => _monaco.Promise.wrap(import('./redis'))
+	loader: () => import('./redis')
 });

src/redshift/redshift.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'redshift',
 	extensions: [],
 	aliases: ['Redshift', 'redshift'],
-	loader: () => _monaco.Promise.wrap(import('./redshift'))
+	loader: () => import('./redshift')
 });

src/ruby/ruby.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'ruby',
 	extensions: ['.rb', '.rbx', '.rjs', '.gemspec', '.pp'],
 	filenames: ['rakefile'],
 	aliases: ['Ruby', 'rb'],
-	loader: () => _monaco.Promise.wrap(import('./ruby'))
+	loader: () => import('./ruby')
 });

src/ruby/ruby.ts

@@ -31,7 +31,11 @@ export const conf: IRichLanguageConfiguration = {
 		{ open: '(', close: ')' },
 		{ open: '"', close: '"' },
 		{ open: '\'', close: '\'' },
-	]
+	],
+	indentationRules: {
+		increaseIndentPattern: new RegExp('^\\s*((begin|class|(private|protected)\\s+def|def|else|elsif|ensure|for|if|module|rescue|unless|until|when|while|case)|([^#]*\\sdo\\b)|([^#]*=\\s*(case|if|unless)))\\b([^#\\{;]|("|\'|\/).*\\4)*(#.*)?$'),
+		decreaseIndentPattern: new RegExp('^\\s*([}\\]]([,)]?\\s*(#|$)|\\.[a-zA-Z_]\\w*\\b)|(end|rescue|ensure|else|elsif|when)\\b)'),
+	}
 };
 
 /*

src/rust/rust.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'rust',
 	extensions: ['.rs', '.rlib'],
 	aliases: ['Rust', 'rust'],
-	loader: () => _monaco.Promise.wrap(import('./rust'))
+	loader: () => import('./rust')
 });

src/sb/sb.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'sb',
 	extensions: ['.sb'],
 	aliases: ['Small Basic', 'sb'],
-	loader: () => _monaco.Promise.wrap(import('./sb'))
+	loader: () => import('./sb')
 });

src/scheme/scheme.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+    id: 'scheme',
+    extensions: ['.scm', '.ss', '.sch', '.rkt'],
+    aliases: ['scheme', 'Scheme'],
+    loader: () => import('./scheme'),
+});

src/scheme/scheme.test.ts

@@ -0,0 +1,91 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('scheme', [
+	// Keywords
+	[
+		{
+			line: 'define-macro some',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.scheme' },
+				{ startIndex: 12, type: 'white.scheme' },
+				{ startIndex: 13, type: 'variable.scheme' },
+			],
+		},
+	],
+
+	// comments
+	[
+		{
+			line: '; comment',
+			tokens: [{ startIndex: 0, type: 'comment.scheme' }],
+		},
+	],
+	[
+		{
+			line: '#| comment',
+			tokens: [{ startIndex: 0, type: 'comment.scheme' }],
+		},
+		{
+			line: 'multiline',
+			tokens: [{ startIndex: 0, type: 'comment.scheme' }],
+		},
+		{
+			line: '|# cons',
+			tokens: [
+				{ startIndex: 0, type: 'comment.scheme' },
+				{ startIndex: 2, type: 'white.scheme' },
+				{ startIndex: 3, type: 'keyword.scheme' },
+			],
+		},
+	],
+
+	// strings
+	[
+		{
+			line: '"\\n string "',
+			tokens: [
+				{ startIndex: 0, type: 'string.scheme' },
+				{ startIndex: 1, type: 'string.escape.scheme' },
+				{ startIndex: 3, type: 'string.scheme' },
+			],
+		},
+	],
+	[
+		{
+			line: '" string \\',
+			tokens: [{ startIndex: 0, type: 'string.scheme' }],
+		},
+		{
+			line: 'multiline',
+			tokens: [{ startIndex: 0, type: 'string.scheme' }],
+		},
+		{
+			line: ' ',
+			tokens: [
+				// previous line needs to be terminated with \
+				{ startIndex: 0, type: 'white.scheme' },
+			],
+		},
+	],
+
+	// numbers
+	[
+		{
+			line: '1e2',
+			tokens: [{ startIndex: 0, type: 'number.float.scheme' }],
+		},
+	],
+	[
+		{
+			line: '#x03BB',
+			tokens: [{ startIndex: 0, type: 'number.hex.scheme' }],
+		},
+	],
+]);

src/scheme/scheme.ts

@@ -0,0 +1,127 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: ';',
+		blockComment: ['#|', '|#'],
+	},
+
+	brackets: [['(', ')'], ['{', '}'], ['[', ']']],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+	],
+
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+	],
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.scheme',
+
+	brackets: [
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+	],
+
+	keywords: [
+		'case',
+		'do',
+		'let',
+		'loop',
+		'if',
+		'else',
+		'when',
+		'cons',
+		'car',
+		'cdr',
+		'cond',
+		'lambda',
+		'lambda*',
+		'syntax-rules',
+		'format',
+		'set!',
+		'quote',
+		'eval',
+		'append',
+		'list',
+		'list?',
+		'member?',
+		'load',
+	],
+
+	constants: ['#t', '#f'],
+
+	operators: ['eq?', 'eqv?', 'equal?', 'and', 'or', 'not', 'null?'],
+
+	tokenizer: {
+		root: [
+			[/#[xXoObB][0-9a-fA-F]+/, 'number.hex'],
+			[/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'],
+
+			[
+				/(?:\b(?:(define|define-syntax|define-macro))\b)(\s+)((?:\w|\-|\!|\?)*)/,
+				['keyword', 'white', 'variable'],
+			],
+
+			{ include: '@whitespace' },
+			{ include: '@strings' },
+
+			[
+				/[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@constants': 'constant',
+						'@operators': 'operators',
+						'@default': 'identifier',
+					},
+				},
+			],
+		],
+
+		comment: [
+			[/[^\|#]+/, 'comment'],
+			[/#\|/, 'comment', '@push'],
+			[/\|#/, 'comment', '@pop'],
+			[/[\|#]/, 'comment'],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+			[/#\|/, 'comment', '@comment'],
+			[/;.*$/, 'comment'],
+		],
+
+		strings: [
+			[/"$/, 'string', '@popall'],
+			[/"(?=.)/, 'string', '@multiLineString'],
+		],
+
+		multiLineString: [
+			[/[^\\"]+$/, 'string', '@popall'],
+			[/[^\\"]+/, 'string'],
+			[/\\./, 'string.escape'],
+			[/"/, 'string', '@popall'],
+			[/\\$/, 'string']
+		],
+	},
+};

src/scss/scss.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'scss',
 	extensions: ['.scss'],
 	aliases: ['Sass', 'sass', 'scss'],
 	mimetypes: ['text/x-scss', 'text/scss'],
-	loader: () => _monaco.Promise.wrap(import('./scss'))
+	loader: () => import('./scss')
 });

src/shell/shell.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the MIT License. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'shell',
+	extensions: ['.sh', '.bash'],
+	aliases: ['Shell', 'sh'],
+	loader: () => import('./shell'),
+});

src/shell/shell.test.ts

@@ -0,0 +1,291 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('shell', [
+	// Keywords
+	[{
+		line: 'if while',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.shell' },
+			{ startIndex: 2, type: 'white.shell' },
+			{ startIndex: 3, type: 'keyword.shell' }
+		]
+	}],
+
+	// Predefined & attribute
+	[{
+		line: 'ps -aux | grep code',
+		tokens: [
+			{ startIndex: 0, type: 'type.identifier.shell' },
+			{ startIndex: 2, type: 'white.shell' },
+			{ startIndex: 3, type: 'attribute.name.shell' },
+			{ startIndex: 7, type: 'white.shell' },
+			{ startIndex: 8, type: 'delimiter.shell' },
+			{ startIndex: 9, type: 'white.shell' },
+			{ startIndex: 10, type: 'type.identifier.shell' },
+			{ startIndex: 14, type: 'white.shell' },
+			{ startIndex: 15, type: '' },
+		]
+	}],
+
+	[{
+		line: '# comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.shell' }
+		]
+	}, {
+		line: 'cd tree',
+		tokens: [
+			{ startIndex: 0, type: 'type.identifier.shell' },
+			{ startIndex: 2, type: 'white.shell' },
+			{ startIndex: 3, type: '' }
+		]
+	}],
+
+	// Shebang
+	[{
+		line: '#!/bin/env bash',
+		tokens: [
+			{ startIndex: 0, type: 'metatag.shell' }
+		]
+	}],
+
+	// Comments
+	[{
+		line: '#',
+		tokens: [
+			{ startIndex: 0, type: 'comment.shell' }
+		]
+	}],
+
+	[{
+		line: '# a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.shell' }
+		]
+	}],
+
+	[{
+		line: '    # a comment',
+		tokens: [
+			{ startIndex: 0, type: 'white.shell' },
+			{ startIndex: 4, type: 'comment.shell' }
+		]
+	}],
+
+	// numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.shell' }
+		]
+	}],
+
+	[{
+		line: '0.0',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.shell' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.shell' }
+		]
+	}],
+
+	[{
+		line: '23.5',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.shell' }
+		]
+	}],
+
+	[{
+		line: '23.5e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.shell' }
+		]
+	}],
+
+	[{
+		line: '23.5E3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.shell' }
+		]
+	}],
+
+	[{
+		line: '1.72e-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.shell' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.shell' },
+			{ startIndex: 1, type: 'delimiter.shell' },
+			{ startIndex: 2, type: 'number.shell' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.shell' },
+			{ startIndex: 3, type: 'delimiter.shell' },
+			{ startIndex: 4, type: 'number.shell' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.shell' },
+			{ startIndex: 1, type: 'white.shell' },
+			{ startIndex: 2, type: 'delimiter.shell' },
+			{ startIndex: 3, type: 'white.shell' },
+			{ startIndex: 4, type: 'number.shell' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: "'test string'",
+		tokens: [
+			{ startIndex: 0, type: 'string.shell' }
+		]
+	}],
+
+	[{
+		line: '"test string"',
+		tokens: [
+			{ startIndex: 0, type: 'string.shell' }
+		]
+	}],
+
+	[{
+		line: "'test",
+		tokens: [
+			{ startIndex: 0, type: 'string.shell' }
+		],
+	}, {
+		line: '',
+		tokens: [
+
+		]
+	}, {
+		line: "string'",
+		tokens: [
+			{ startIndex: 0, type: 'string.shell' }
+		]
+	}],
+
+	[{
+		line: '"test',
+		tokens: [
+			{ startIndex: 0, type: 'string.shell' }
+		],
+	}, {
+		line: '',
+		tokens: [
+
+		]
+	}, {
+		line: 'string"',
+		tokens: [
+			{ startIndex: 0, type: 'string.shell' }
+		]
+	}],
+
+	// Parameters
+	[{
+		line: '$1',
+		tokens: [
+			{ startIndex: 0, type: 'variable.predefined.shell' }
+		]
+	}],
+
+	[{
+		line: '$a',
+		tokens: [
+			{ startIndex: 0, type: 'variable.shell' }
+		]
+	}],
+
+	[{
+		line: '${string:position}',
+		tokens: [
+			{ startIndex: 0, type: 'variable.shell' },
+			{ startIndex: 8, type: 'delimiter.shell' },
+			{ startIndex: 9, type: 'variable.shell' }
+		]
+	}],
+
+	[{
+		line: '$(pwd)',
+		tokens: [
+			{ startIndex: 0, type: 'variable.shell' },
+		]
+	}],
+
+	[{
+		line: 'echo $hello | less',
+		tokens: [
+			{ startIndex: 0, type: 'type.identifier.shell' },
+			{ startIndex: 4, type: 'white.shell' },
+			{ startIndex: 5, type: 'variable.shell' },
+			{ startIndex: 11, type: 'white.shell' },
+			{ startIndex: 12, type: 'delimiter.shell' },
+			{ startIndex: 13, type: 'white.shell' },
+			{ startIndex: 14, type: '' }
+		]
+	}],
+
+	// HereDoc
+	[{
+		line: '<< word',
+		tokens: [
+			{ startIndex: 0, type: 'constants.shell' },
+			{ startIndex: 2, type: 'white.shell' },
+			{ startIndex: 3, type: 'string.heredoc.shell' }
+		]
+	}],
+
+	[{
+		line: '<<- "word"',
+		tokens: [
+			{ startIndex: 0, type: 'constants.shell' },
+			{ startIndex: 3, type: 'white.shell' },
+			{ startIndex: 4, type: 'string.heredoc.delimiter.shell' },
+			{ startIndex: 5, type: 'string.heredoc.shell' },
+			{ startIndex: 9, type: 'string.heredoc.delimiter.shell' },
+		]
+	}],
+
+	[{
+		line: '<<< word',
+		tokens: [
+			{ startIndex: 0, type: 'constants.shell' },
+			{ startIndex: 3, type: 'white.shell' },
+			{ startIndex: 4, type: 'string.heredoc.shell' }
+		]
+	}],
+
+	[{
+		line: 'echo $( echo "hi" )',
+		tokens: [
+			{ startIndex: 0, type: 'type.identifier.shell' },
+			{ startIndex: 4, type: 'white.shell' },
+			{ startIndex: 5, type: 'variable.shell' }
+		]
+	}],
+])

src/shell/shell.ts

@@ -0,0 +1,230 @@
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the MIT License. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+	},
+	brackets: [['{', '}'], ['[', ']'], ['(', ')']],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '`', close: '`' },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '`', close: '`' },
+	],
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.shell',
+
+	brackets: [
+		{ token: 'delimiter.bracket', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+	],
+
+	keywords: [
+		'if',
+		'then',
+		'do',
+		'else',
+		'elif',
+		'while',
+		'until',
+		'for',
+		'in',
+		'esac',
+		'fi',
+		'fin',
+		'fil',
+		'done',
+		'exit',
+		'set',
+		'unset',
+		'export',
+		'function',
+	],
+
+	builtins: [
+		'ab',
+		'awk',
+		'bash',
+		'beep',
+		'cat',
+		'cc',
+		'cd',
+		'chown',
+		'chmod',
+		'chroot',
+		'clear',
+		'cp',
+		'curl',
+		'cut',
+		'diff',
+		'echo',
+		'find',
+		'gawk',
+		'gcc',
+		'get',
+		'git',
+		'grep',
+		'hg',
+		'kill',
+		'killall',
+		'ln',
+		'ls',
+		'make',
+		'mkdir',
+		'openssl',
+		'mv',
+		'nc',
+		'node',
+		'npm',
+		'ping',
+		'ps',
+		'restart',
+		'rm',
+		'rmdir',
+		'sed',
+		'service',
+		'sh',
+		'shopt',
+		'shred',
+		'source',
+		'sort',
+		'sleep',
+		'ssh',
+		'start',
+		'stop',
+		'su',
+		'sudo',
+		'svn',
+		'tee',
+		'telnet',
+		'top',
+		'touch',
+		'vi',
+		'vim',
+		'wall',
+		'wc',
+		'wget',
+		'who',
+		'write',
+		'yes',
+		'zsh',
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?&|+\-*\/\^;\.,]+/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			{ include: '@whitespace' },
+
+			[
+				/[a-zA-Z]\w*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@builtins': 'type.identifier',
+						'@default': ''
+					},
+				},
+			],
+
+			{ include: '@strings' },
+			{ include: '@parameters' },
+			{ include: '@heredoc' },
+
+			[/[{}\[\]()]/, '@brackets'],
+
+			[/-+\w+/, 'attribute.name'],
+
+			[/@symbols/, 'delimiter'],
+
+			{ include: '@numbers' },
+
+			[/[,;]/, 'delimiter'],
+		],
+
+		whitespace: [
+			[/\s+/, 'white'],
+			[/(^#!.*$)/, 'metatag'],
+			[/(^#.*$)/, 'comment'],
+		],
+
+		numbers: [
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+/, 'number'],
+		],
+
+		// Recognize strings, including those broken across lines
+		strings: [
+			[/'/, 'string', '@stringBody'],
+			[/"/, 'string', '@dblStringBody']
+		],
+		stringBody: [
+			[/'/, 'string', '@popall'],
+			[/./, 'string'],
+		],
+		dblStringBody: [
+			[/"/, 'string', '@popall'],
+			[/./, 'string'],
+		],
+
+		heredoc: [
+			[/(<<[-<]?)(\s*)(['"`]?)([\w\-]+)(['"`]?)/, ['constants', 'white', 'string.heredoc.delimiter', 'string.heredoc', 'string.heredoc.delimiter']]
+		],
+
+		parameters: [
+			[/\$\d+/, 'variable.predefined'],
+			[/\$\w+/, 'variable'],
+			[/\$[*@#?\-$!0_]/, 'variable'],
+			[/\$'/, 'variable', '@parameterBodyQuote'],
+			[/\$"/, 'variable', '@parameterBodyDoubleQuote'],
+			[/\$\(/, 'variable', '@parameterBodyParen'],
+			[/\$\{/, 'variable', '@parameterBodyCurlyBrace'],
+		],
+		parameterBodyQuote: [
+			[/[^#:%*@\-!_']+/, 'variable'],
+			[/[#:%*@\-!_]/, 'delimiter'],
+			[/[']/, 'variable', '@pop'],
+		],
+		parameterBodyDoubleQuote: [
+			[/[^#:%*@\-!_"]+/, 'variable'],
+			[/[#:%*@\-!_]/, 'delimiter'],
+			[/["]/, 'variable', '@pop'],
+		],
+		parameterBodyParen: [
+			[/[^#:%*@\-!_)]+/, 'variable'],
+			[/[#:%*@\-!_]/, 'delimiter'],
+			[/[)]/, 'variable', '@pop'],
+		],
+		parameterBodyCurlyBrace: [
+			[/[^#:%*@\-!_}]+/, 'variable'],
+			[/[#:%*@\-!_]/, 'delimiter'],
+			[/[}]/, 'variable', '@pop'],
+		],
+	}
+};

src/solidity/solidity.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'sol',
 	extensions: ['.sol'],
 	aliases: ['sol', 'solidity', 'Solidity'],
-	loader: () => _monaco.Promise.wrap(import('./solidity'))
+	loader: () => import('./solidity')
 });

src/solidity/solidity.ts

@@ -43,6 +43,7 @@ export const language = <ILanguage>{
 		'struct',
 		'function',
 		'modifier',
+		'constructor',
 		//Built-in types
 		'address',
 		'string',

src/sql/sql.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'sql',
 	extensions: ['.sql'],
 	aliases: ['SQL'],
-	loader: () => _monaco.Promise.wrap(import('./sql'))
+	loader: () => import('./sql')
 });

src/st/st.contribution.ts

@@ -6,12 +6,9 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'st',
 	extensions: ['.st', '.iecst', '.iecplc', '.lc3lib'],
 	aliases: ['StructuredText', 'scl', 'stl'],
-	loader: () => _monaco.Promise.wrap(import('./st'))
+	loader: () => import('./st')
 });

src/swift/swift.contribution.ts

@@ -6,13 +6,10 @@
 
 import { registerLanguage } from '../_.contribution';
 
-// Allow for running under nodejs/requirejs in tests
-const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
-
 registerLanguage({
 	id: 'swift',
 	aliases: ['Swift', 'swift'],
 	extensions: ['.swift'],
 	mimetypes: ['text/swift'],
-	loader: () => _monaco.Promise.wrap(import('./swift'))
+	loader: () => import('./swift')
 });

src/tcl/tcl.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import {registerLanguage} from '../_.contribution';
+
+registerLanguage({
+	id: 'tcl',
+	extensions: ['.tcl'],
+	aliases: ['tcl', 'Tcl', 'tcltk', 'TclTk', 'tcl/tk', 'Tcl/Tk'],
+	loader: () => import('./tcl')
+});

src/tcl/tcl.test.ts

@@ -0,0 +1,100 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from '../test/testRunner';
+
+testTokenization('tcl', [
+	// Definitions
+	[{
+		line: 'set var 1',
+		tokens: [
+			{startIndex: 0, type: 'keyword.flow.tcl'}, // `set`
+			{startIndex: 3, type: 'source.tcl'},       // ` `
+			{startIndex: 4, type: 'type.tcl'},         // `var`
+			{startIndex: 7, type: 'white.tcl'},        // ` `
+			{startIndex: 8, type: 'number.tcl'}        // `1`
+		]
+	}],
+	
+	[{
+		line: 'proc func {a b} {}',
+		tokens: [
+			{startIndex: 0, type: 'keyword.flow.tcl'},     // `proc`
+			{startIndex: 4, type: 'source.tcl'},           // ` `
+			{startIndex: 5, type: 'type.tcl'},             // `func`
+			{startIndex: 9, type: 'white.tcl'},           // ` `
+			{startIndex: 10, type: 'delimiter.curly.tcl'}, // `{`
+			{startIndex: 11, type: 'operator.scss.tcl'},   // `a`
+			{startIndex: 12, type: 'white.tcl'},          // ` `
+			{startIndex: 13, type: 'operator.scss.tcl'},   // `b`
+			{startIndex: 14, type: 'delimiter.curly.tcl'}, // `}`
+			{startIndex: 15, type: 'white.tcl'},          // ` `
+			{startIndex: 16, type: 'delimiter.curly.tcl'}  // `{}`
+		]
+	}],
+	
+	// Keywords
+	[{
+		line: 'if 1 return',
+		tokens: [
+			{startIndex: 0, type: 'keyword.tcl'},
+			{startIndex: 2, type: 'white.tcl'},
+			{startIndex: 3, type: 'number.tcl'},
+			{startIndex: 4, type: 'white.tcl'},
+			{startIndex: 5, type: 'keyword.tcl'}
+		]
+	}],
+	
+	
+	// Variables
+	[{
+		line: '$var1 $::var2 $$var3 ${var 4} $::{var 5}',
+		tokens: [
+			{startIndex: 0, type: 'type.identifier.tcl'},
+			{startIndex: 5, type: 'white.tcl'},
+			{startIndex: 6, type: 'type.identifier.tcl'},
+			{startIndex: 13, type: 'white.tcl'},
+			{startIndex: 14, type: 'type.identifier.tcl'},
+			{startIndex: 20, type: 'white.tcl'},
+			{startIndex: 21, type: 'identifier.tcl'},
+			{startIndex: 23, type: 'type.identifier.tcl'},
+			{startIndex: 28, type: 'identifier.tcl'},
+			{startIndex: 29, type: 'white.tcl'},
+			{startIndex: 30, type: 'identifier.tcl'},
+			{startIndex: 34, type: 'type.identifier.tcl'},
+			{startIndex: 39, type: 'identifier.tcl'}
+		]
+	}],
+	
+	// Function and variable string interpolation
+	[{
+		line: 'puts "$var1 + ${var 2} = [expr $var1 + ${var 2}]"',
+		tokens: [
+			{startIndex: 0, type: 'variable.tcl'},
+			{startIndex: 4, type: 'white.tcl'},
+			{startIndex: 5, type: 'string.quote.tcl'},
+			{startIndex: 6, type: 'type.identifier.tcl'},
+			{startIndex: 11, type: 'string.tcl'},
+			{startIndex: 14, type: 'identifier.tcl'},
+			{startIndex: 16, type: 'type.identifier.tcl'},
+			{startIndex: 21, type: 'identifier.tcl'},
+			{startIndex: 22, type: 'string.tcl'},
+			{startIndex: 25, type: 'delimiter.square.tcl'},
+			{startIndex: 26, type: 'keyword.tcl'},
+			{startIndex: 30, type: 'white.tcl'},
+			{startIndex: 31, type: 'type.identifier.tcl'},
+			{startIndex: 36, type: 'white.tcl'},
+			{startIndex: 37, type: 'operator.tcl'},
+			{startIndex: 38, type: 'white.tcl'},
+			{startIndex: 39, type: 'identifier.tcl'},
+			{startIndex: 41, type: 'type.identifier.tcl'},
+			{startIndex: 46, type: 'identifier.tcl'},
+			{startIndex: 47, type: 'delimiter.square.tcl'},
+			{startIndex: 48, type: 'string.quote.tcl'}
+		]
+	}]
+]);

src/tcl/tcl.ts

@@ -0,0 +1,159 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{open: '{', close: '}'},
+		{open: '[', close: ']'},
+		{open: '(', close: ')'},
+		{open: '"', close: '"'},
+		{open: '\'', close: '\''},
+	],
+	surroundingPairs: [
+		{open: '{', close: '}'},
+		{open: '[', close: ']'},
+		{open: '(', close: ')'},
+		{open: '"', close: '"'},
+		{open: '\'', close: '\''},
+	]
+};
+
+export const language = <ILanguage>{
+	tokenPostfix: '.tcl',
+	
+	specialFunctions: [
+		'set', 'unset', 'rename', 'variable', 'proc', 'coroutine',
+		'foreach',
+		'incr', 'append',
+		'lappend', 'linsert', 'lreplace'
+	],
+
+	mainFunctions: [
+		'if', 'then', 'elseif', 'else', 'case', 'switch', 'while', 'for',
+		'break', 'continue', 'return',
+		'package', 'namespace',
+		'catch', 'exit',
+		'eval', 'expr', 'uplevel', 'upvar'
+	],
+
+	builtinFunctions: [
+		'file', 'info', 'concat', 'join', 'lindex',
+		'list', 'llength', 'lrange', 'lsearch', 'lsort', 'split',
+		'array', 'parray', 'binary', 'format', 'regexp', 'regsub', 'scan', 'string',
+		'subst', 'dict', 'cd', 'clock', 'exec', 'glob', 'pid', 'pwd', 'close', 'eof', 'fblocked',
+		'fconfigure', 'fcopy', 'fileevent', 'flush', 'gets', 'open', 'puts', 'read', 'seek',
+		'socket', 'tell', 'interp', 'after', 'auto_execok',
+		'auto_load', 'auto_mkindex', 'auto_reset', 'bgerror', 'error',
+		'global', 'history', 'load', 'source', 'time', 'trace',
+		'unknown', 'unset', 'update', 'vwait', 'winfo', 'wm', 'bind', 'event',
+		'pack', 'place', 'grid', 'font', 'bell', 'clipboard', 'destroy', 'focus', 'grab', 'lower',
+		'option', 'raise', 'selection', 'send', 'tk', 'tkwait', 'tk_bisque', 'tk_focusNext',
+		'tk_focusPrev', 'tk_focusFollowsMouse', 'tk_popup', 'tk_setPalette'
+	],
+
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	
+	brackets: [
+		{open: '(', close: ')', token: 'delimiter.parenthesis'},
+		{open: '{', close: '}', token: 'delimiter.curly'},
+		{open: '[', close: ']', token: 'delimiter.square'}
+	],
+	
+	escapes: /\\(?:[abfnrtv\\"'\[\]\{\};\$]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	variables: /(?:\$+(?:(?:\:\:?)?[a-zA-Z_]\w*)+)/,
+
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+
+			[/[a-zA-Z_]\w*/, {cases: {
+				'@specialFunctions': {token: 'keyword.flow', next: '@specialFunc'},
+				'@mainFunctions': 'keyword',
+				'@builtinFunctions': 'variable',
+				'@default': 'operator.scss'
+			}}],
+
+			[/\s+\-+(?!\d|\.)\w*|{\*}/, 'metatag'],
+
+			// whitespace
+			{include: '@whitespace'},
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'operator'],
+			[/\$+(?:\:\:)?\{/, {token: 'identifier', next: '@nestedVariable'}],
+			[/@variables/, 'type.identifier'],
+			[/\.(?!\d|\.)[\w\-]*/, 'operator.sql'],
+
+			// numbers
+			[/\d+(\.\d+)?/, 'number'],
+			[/\d+/, 'number'],
+
+			// delimiter
+			[/;/, 'delimiter'],
+
+			// strings
+			[/"/,  {token: 'string.quote', bracket: '@open', next: '@dstring'}],
+			[/'/,  {token: 'string.quote', bracket: '@open', next: '@sstring'}]
+		],
+
+		dstring: [
+			[/\[/, {token: '@brackets', next: '@nestedCall'}],
+			[/\$+(?:\:\:)?\{/, {token: 'identifier', next: '@nestedVariable'}],
+			[/@variables/, 'type.identifier'],
+			[/[^\\$\[\]"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/"/,					{token: 'string.quote', bracket: '@close', next: '@pop'}],
+		],
+
+		sstring: [
+			[/\[/, {token: '@brackets', next: '@nestedCall'}],
+			[/\$+(?:\:\:)?\{/, {token: 'identifier', next: '@nestedVariable'}],
+			[/@variables/, 'type.identifier'],
+			[/[^\\$\[\]']+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/'/,					{token: 'string.quote', bracket: '@close', next: '@pop'}]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+			[/#.*\\$/, 			{token: 'comment', next: '@newlineComment'}],
+			[/#.*(?!\\)$/, 'comment']
+		],
+
+		newlineComment: [
+			[/.*\\$/, 'comment'],
+			[/.*(?!\\)$/, {token: 'comment', next: '@pop'}]
+		],
+
+		nestedVariable: [
+			[/[^\{\}\$]+/, 'type.identifier'],
+			[/\}/, {token: 'identifier', next: '@pop'}]
+		],
+
+		nestedCall: [
+			[/\[/, {token: '@brackets', next: '@nestedCall'}],
+			[/\]/, {token: '@brackets', next: '@pop'}],
+			{include: 'root'}
+		],
+
+		specialFunc: [
+			[/"/, {token: 'string', next: '@dstring'}],
+			[/'/, {token: 'string', next: '@sstring'}],
+			[/(?:(?:\:\:?)?[a-zA-Z_]\w*)+/, {token: 'type', next: '@pop'}],
+		]
+	}
+};