Initial release

.gitignore

@@ -0,0 +1,3 @@
+/node_modules/
+/out/
+/release/

.npmignore

@@ -0,0 +1,7 @@
+/.vscode/
+/out/
+/src/
+/test/
+/gulpfile.js
+/tsconfig.json
+/.npmignore

.vscode/settings.json

@@ -0,0 +1,9 @@
+// Place your settings in this file to overwrite default and user settings.
+{
+	"files.trimTrailingWhitespace": true,
+	"search.exclude": {
+		"**/node_modules": true,
+		"**/release": true,
+		"**/out": true
+	}
+}

LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,37 @@
-# monaco-languages
-Basic colorization support for many Monaco Editor languages.
+# Monaco Languages
+
+Colorization and configuration supports for multiple languages for the Monaco Editor:
+
+![monaco-languages](https://cloud.githubusercontent.com/assets/5047891/15938606/1fd4bac6-2e74-11e6-8839-d455da8bc8a7.gif)
+
+* bat
+* coffee script
+* cpp
+* csharp
+* fsharp
+* go
+* ini
+* jade
+* lua
+* objective-c
+* powershell
+* python
+* r
+* ruby
+* sql
+* swift
+* vb
+* xml
+
+## Installing
+
+This npm module is bundled and distributed in the [monaco-editor](https://www.npmjs.com/package/monaco-editor) npm module.
+
+## Dev:
+
+* compile with `npm run watch`
+* test with `npm run test`
+* bundle with `npm run prepublish`
+
+## License
+[MIT](https://github.com/Microsoft/monaco-typescript/blob/master/LICENSE.md)

ThirdPartyNotices.txt

@@ -0,0 +1,34 @@
+THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
+Do Not Translate or Localize
+
+This project incorporates components from the projects listed below. The original copyright notices and the licenses
+under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted
+herein, whether by implication, estoppel or otherwise.
+
+
+
+
+%% vscode-swift version 0.0.1 (https://github.com/owensd/vscode-swift)
+=========================================
+The MIT License (MIT)
+
+Copyright (c) 2015 David Owens II
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+=========================================
+END OF vscode-swift NOTICES AND INFORMATION

gulpfile.js

@@ -0,0 +1,148 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+var gulp = require('gulp');
+var tsb = require('gulp-tsb');
+var assign = require('object-assign');
+var fs = require('fs');
+var path = require('path');
+var merge = require('merge-stream');
+var rjs = require('gulp-requirejs');
+var uglify = require('gulp-uglify');
+var rimraf = require('rimraf');
+var es = require('event-stream');
+
+gulp.task('clean-release', function(cb) { rimraf('release', { maxBusyTries: 1 }, cb); });
+gulp.task('release', ['clean-release','compile'], function() {
+
+	var sha1 = getGitVersion(__dirname);
+	var semver = require('./package.json').version;
+	var headerVersion = semver + '(' + sha1 + ')';
+
+	var BUNDLED_FILE_HEADER = [
+		'/*!-----------------------------------------------------------------------------',
+		' * Copyright (c) Microsoft Corporation. All rights reserved.',
+		' * monaco-languages version: ' + headerVersion,
+		' * Released under the MIT license',
+		' * https://github.com/Microsoft/monaco-languages/blob/master/LICENSE.md',
+		' *-----------------------------------------------------------------------------*/',
+		''
+	].join('\n');
+
+	function bundleOne(moduleId, exclude) {
+		return rjs({
+			baseUrl: '/out/',
+			name: 'vs/basic-languages/' + moduleId,
+			out: moduleId + '.js',
+			exclude: exclude,
+			paths: {
+				'vs/basic-languages': __dirname + '/out'
+			}
+		})
+	}
+
+	return merge(
+			bundleOne('src/monaco.contribution'),
+			bundleOne('src/bat'),
+			bundleOne('src/coffee'),
+			bundleOne('src/cpp'),
+			bundleOne('src/csharp'),
+			bundleOne('src/dockerfile'),
+			bundleOne('src/fsharp'),
+			bundleOne('src/go'),
+			bundleOne('src/ini'),
+			bundleOne('src/jade'),
+			bundleOne('src/java'),
+			bundleOne('src/lua'),
+			bundleOne('src/objective-c'),
+			bundleOne('src/powershell'),
+			bundleOne('src/python'),
+			bundleOne('src/r'),
+			bundleOne('src/ruby'),
+			bundleOne('src/sql'),
+			bundleOne('src/swift'),
+			bundleOne('src/vb'),
+			bundleOne('src/xml')
+		)
+		.pipe(uglify({
+			preserveComments: 'some'
+		}))
+		.pipe(es.through(function(data) {
+			data.contents = new Buffer(
+				BUNDLED_FILE_HEADER
+				+ data.contents.toString()
+			);
+			this.emit('data', data);
+		}))
+		.pipe(gulp.dest('./release/'));
+});
+
+
+var compilation = tsb.create(assign({ verbose: true }, require('./tsconfig.json').compilerOptions));
+
+var tsSources = require('./tsconfig.json').filesGlob;
+
+function compileTask() {
+	return merge(
+		gulp.src(tsSources, { base: '.' }).pipe(compilation())
+	)
+	.pipe(gulp.dest('out'));
+}
+gulp.task('clean-out', function(cb) { rimraf('out', { maxBusyTries: 1 }, cb); });
+gulp.task('compile', ['clean-out'], compileTask);
+gulp.task('compile-without-clean', compileTask);
+gulp.task('watch', ['compile'], function() {
+	gulp.watch(tsSources, ['compile-without-clean']);
+});
+
+function getGitVersion(repo) {
+	var git = path.join(repo, '.git');
+	var headPath = path.join(git, 'HEAD');
+	var head;
+
+	try {
+		head = fs.readFileSync(headPath, 'utf8').trim();
+	} catch (e) {
+		return void 0;
+	}
+
+	if (/^[0-9a-f]{40}$/i.test(head)) {
+		return head;
+	}
+
+	var refMatch = /^ref: (.*)$/.exec(head);
+
+	if (!refMatch) {
+		return void 0;
+	}
+
+	var ref = refMatch[1];
+	var refPath = path.join(git, ref);
+
+	try {
+		return fs.readFileSync(refPath, 'utf8').trim();
+	} catch (e) {
+		// noop
+	}
+
+	var packedRefsPath = path.join(git, 'packed-refs');
+	var refsRaw;
+
+	try {
+		refsRaw = fs.readFileSync(packedRefsPath, 'utf8').trim();
+	} catch (e) {
+		return void 0;
+	}
+
+	var refsRegex = /^([0-9a-f]{40})\s+(.+)$/gm;
+	var refsMatch;
+	var refs = {};
+
+	while (refsMatch = refsRegex.exec(refsRaw)) {
+		refs[refsMatch[2]] = refsMatch[1];
+	}
+
+	return refs[ref];
+}

package.json

@@ -0,0 +1,34 @@
+{
+  "name": "monaco-languages",
+  "version": "0.1.0",
+  "description": "Bundle of many languages for the Monaco Editor.",
+  "scripts": {
+    "test": "node_modules/.bin/mocha",
+    "watch": "node_modules/.bin/gulp watch",
+    "prepublish": "node_modules/.bin/gulp release"
+  },
+  "author": "Microsoft Corporation",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Microsoft/monaco-languages"
+  },
+  "bugs": {
+    "url": "https://github.com/Microsoft/monaco-languages/issues"
+  },
+  "devDependencies": {
+    "event-stream": "^3.3.2",
+    "gulp": "^3.9.1",
+    "gulp-requirejs": "^0.1.3",
+    "gulp-tsb": "^1.10.4",
+    "gulp-uglify": "^1.5.3",
+    "jsdom-no-contextify": "^3.1.0",
+    "merge-stream": "^1.0.0",
+    "mocha": "^2.5.3",
+    "monaco-editor-core": "0.3.1",
+    "object-assign": "^4.1.0",
+    "rimraf": "^2.5.2",
+    "typescript": "^1.8.10",
+    "typescript-with-globs": "^0.1.4"
+  }
+}

src/bat.ts

@@ -0,0 +1,106 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: 'REM'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [
+	// 		{
+	// 			openTrigger: 'l',
+	// 			open: /setlocal$/i,
+	// 			closeComplete: 'endlocal',
+	// 			matchCase: true,
+	// 			closeTrigger: 'l',
+	// 			close: /endlocal$/i,
+	// 			tokenType: 'keyword.tag-setlocal'
+	// 		}
+	// 	],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.bat',
+
+	brackets: [
+		{ token: 'punctuation.bracket', open: '{', close: '}' },
+		{ token: 'punctuation.parenthesis', open: '(', close: ')' },
+		{ token: 'punctuation.square', open: '[', close: ']' }
+	],
+
+	keywords: /call|defined|echo|errorlevel|exist|for|goto|if|pause|set|shift|start|title|not|pushd|popd/,
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?&|+\-*\/\^;\.,]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			[/^(\s*)(rem(?:\s.*|))$/, ['','comment']],
+
+			[/(\@?)(@keywords)(?!\w)/, [{token:'support.function'}, {token:'support.function.$2'}]],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// blocks
+			[/setlocal(?!\w)/, { token: 'support.function.tag-setlocal', bracket: '@open' }],
+			[/endlocal(?!\w)/, { token: 'support.function.tag-setlocal', bracket: '@close' }],
+
+			// words
+			[/[a-zA-Z_]\w*/, ''],
+
+			// labels
+			[/:\w*/, 'metatag'],
+
+			// variables
+			[/%[^%]+%/, 'variable'],
+			[/%%[\w]+(?!\w)/, 'variable'],
+
+			// punctuations
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'punctuation'],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'constant.numeric.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'constant.numeric.hex'],
+			[/\d+/, 'constant.numeric'],
+
+			// punctuation: after number because of .\d floats
+			[/[;,.]/, 'punctuation'],
+
+			// strings:
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		string: [
+			[/[^\\"'%]+/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/%[\w ]+%/, 'variable'],
+			[/%%[\w]+(?!\w)/, 'variable'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+							'@default': 'string' }} ],
+			[/$/, 'string', '@popall']
+		],
+
+	}
+};

src/coffee.ts

@@ -0,0 +1,157 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\$\-\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		blockComment: ['###', '###'],
+		lineComment: '#'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [
+	// 		{ open: /for$/ }, { open: /while$/ },	{ open: /loop$/ }, { open: /if$/ }, { open: /unless$/ },
+	// 		{ open: /else$/ }, { open: /switch$/ }, { open: /try$/ }, { open: /catch$/ }, { open: /finally$/ },
+	// 		{ open: /class$/ }, { open: /->$/ }
+	// 	],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.coffee',
+
+	brackets: [
+		{ open:'{', close:'}', token:'delimiter.curly'},
+		{ open:'[', close:']', token:'delimiter.square'},
+		{ open:'(', close:')', token:'delimiter.parenthesis'}
+	],
+
+	regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/,
+
+	keywords: [
+		'and', 'or', 'is', 'isnt', 'not', 'on', 'yes', '@', 'no', 'off',
+		'true', 'false', 'null', 'this',
+		'new', 'delete', 'typeof', 'in', 'instanceof',
+		'return', 'throw', 'break', 'continue', 'debugger',
+		'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally',
+		'class', 'extends', 'super',
+		'undefined', 'then', 'unless', 'until', 'loop', 'of', 'by', 'when'
+		],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?&%|+\-*\/\^\.,\:]+/,
+	escapes:  /\\(?:[abfnrtv\\"'$]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// identifiers and keywords
+			[/\@[a-zA-Z_]\w*/, 'variable.predefined'],
+			[/[a-zA-Z_]\w*/, { cases: {
+				'this': 'variable.predefined',
+				'@keywords': { token: 'keyword.$0' },
+				'@default': ''
+				}
+			}],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// Comments
+			[/###/, 'comment', '@comment'],
+			[/#.*$/, 'comment'],
+
+			// regular expressions
+			['///', { token: 'regexp', next: '@hereregexp' }],
+
+			[/^(\s*)(@regEx)/, ['', 'regexp']],
+			[/(\()(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\,)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\=)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\:)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\[)(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\!)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\&)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\|)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\?)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\{)(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\;)(\s*)(@regEx)/, ['', '', 'regexp']],
+
+
+			// delimiters
+			[/}/, { cases: {
+					'$S2==interpolatedstring' : { token: 'string', next: '@pop' }
+					,	'@default'   : '@brackets' } }],
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d+\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/0[0-7]+(?!\d)/, 'number.octal'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[,.]/, 'delimiter'],
+
+			// strings:
+			[/"""/, 'string', '@herestring."""'],
+			[/'''/,  'string', '@herestring.\'\'\''],
+			[/"/,  { cases: { '@eos': 'string', '@default': {token:'string', next:'@string."'} }} ],
+			[/'/, { cases: { '@eos': 'string', '@default': {token:'string', next:'@string.\''} }} ],
+		],
+
+		string: [
+			[/[^"'\#\\]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\./, 'string.escape.invalid'],
+			[/\./, 'string.escape.invalid'],
+
+			[/#{/,  { cases: { '$S2=="': { token: 'string', next: 'root.interpolatedstring' }, '@default': 'string' }}],
+
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' }, '@default': 'string' }} ],
+			[/#/, 'string']
+		],
+
+		herestring: [
+			[/("""|''')/, { cases: { '$1==$S2': { token: 'string', next: '@pop' }, '@default': 'string' } }],
+			[/[^#\\'"]+/,'string' ],
+			[/['"]+/,'string' ],
+			[/@escapes/, 'string.escape'],
+			[/\./, 'string.escape.invalid'],
+
+			[/#{/, { token: 'string.quote', bracket: '@open', next: 'root.interpolatedstring' } ],
+			[/#/, 'string']
+		],
+
+		comment: [
+			[/[^#]+/, 'comment', ],
+			[/###/, 'comment', '@pop'],
+			[/#/, 'comment' ],
+		],
+
+		hereregexp: [
+			[/[^\\\/#]/, 'regexp'],
+			[/\\./, 'regexp'],
+			[/#.*$/, 'comment'],
+			['///[igm]*', { token: 'regexp', next: '@pop' }],
+			[/\//, 'regexp'],
+		],
+	},
+};

src/cpp.ts

@@ -0,0 +1,306 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.cpp',
+
+	brackets: [
+		{ token: 'delimiter.curly', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+		{ token: 'delimiter.angle', open: '<', close: '>' }
+	],
+
+	keywords: [
+		'abstract',
+		'amp',
+		'array',
+		'auto',
+		'bool',
+		'break',
+		'case',
+		'catch',
+		'char',
+		'class',
+		'const',
+		'constexpr',
+		'const_cast',
+		'continue',
+		'cpu',
+		'decltype',
+		'default',
+		'delegate',
+		'delete',
+		'do',
+		'double',
+		'dynamic_cast',
+		'each',
+		'else',
+		'enum',
+		'event',
+		'explicit',
+		'export',
+		'extern',
+		'false',
+		'final',
+		'finally',
+		'float',
+		'for',
+		'friend',
+		'gcnew',
+		'generic',
+		'goto',
+		'if',
+		'in',
+		'initonly',
+		'inline',
+		'int',
+		'interface',
+		'interior_ptr',
+		'internal',
+		'literal',
+		'long',
+		'mutable',
+		'namespace',
+		'new',
+		'noexcept',
+		'nullptr',
+		'__nullptr',
+		'operator',
+		'override',
+		'partial',
+		'pascal',
+		'pin_ptr',
+		'private',
+		'property',
+		'protected',
+		'public',
+		'ref',
+		'register',
+		'reinterpret_cast',
+		'restrict',
+		'return',
+		'safe_cast',
+		'sealed',
+		'short',
+		'signed',
+		'sizeof',
+		'static',
+		'static_assert',
+		'static_cast',
+		'struct',
+		'switch',
+		'template',
+		'this',
+		'thread_local',
+		'throw',
+		'tile_static',
+		'true',
+		'try',
+		'typedef',
+		'typeid',
+		'typename',
+		'union',
+		'unsigned',
+		'using',
+		'virtual',
+		'void',
+		'volatile',
+		'wchar_t',
+		'where',
+		'while',
+
+		'_asm', // reserved word with one underscores
+		'_based',
+		'_cdecl',
+		'_declspec',
+		'_fastcall',
+		'_if_exists',
+		'_if_not_exists',
+		'_inline',
+		'_multiple_inheritance',
+		'_pascal',
+		'_single_inheritance',
+		'_stdcall',
+		'_virtual_inheritance',
+		'_w64',
+
+		'__abstract',  // reserved word with two underscores
+		'__alignof',
+		'__asm',
+		'__assume',
+		'__based',
+		'__box',
+		'__builtin_alignof',
+		'__cdecl',
+		'__clrcall',
+		'__declspec',
+		'__delegate',
+		'__event',
+		'__except',
+		'__fastcall',
+		'__finally',
+		'__forceinline',
+		'__gc',
+		'__hook',
+		'__identifier',
+		'__if_exists',
+		'__if_not_exists',
+		'__inline',
+		'__int128',
+		'__int16',
+		'__int32',
+		'__int64',
+		'__int8',
+		'__interface',
+		'__leave',
+		'__m128',
+		'__m128d',
+		'__m128i',
+		'__m256',
+		'__m256d',
+		'__m256i',
+		'__m64',
+		'__multiple_inheritance',
+		'__newslot',
+		'__nogc',
+		'__noop',
+		'__nounwind',
+		'__novtordisp',
+		'__pascal',
+		'__pin',
+		'__pragma',
+		'__property',
+		'__ptr32',
+		'__ptr64',
+		'__raise',
+		'__restrict',
+		'__resume',
+		'__sealed',
+		'__single_inheritance',
+		'__stdcall',
+		'__super',
+		'__thiscall',
+		'__try',
+		'__try_cast',
+		'__typeof',
+		'__unaligned',
+		'__unhook',
+		'__uuidof',
+		'__value',
+		'__virtual_inheritance',
+		'__w64',
+		'__wchar_t'
+	],
+
+	operators: [
+		'=', '>', '<', '!', '~', '?', ':',
+		'==', '<=', '>=', '!=', '&&', '||', '++', '--',
+		'+', '-', '*', '/', '&', '|', '^', '%', '<<',
+		'>>', '>>>', '+=', '-=', '*=', '/=', '&=', '|=',
+		'^=', '%=', '<<=', '>>=', '>>>='
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /(ll|LL|u|U|l|L)?(ll|LL|u|U|l|L)?/,
+	floatsuffix: /[fFlL]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [[ attributes ]].
+			[/\[\[.*\]\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+									'@default'  : '' } } ],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/0[xX][0-9a-fA-F']*[0-9a-fA-F](@integersuffix)/, 'number.hex'],
+			[/0[0-7']*[0-7](@integersuffix)/, 'number.octal'],
+			[/0[bB][0-1']*[0-1](@integersuffix)/, 'number.binary'],
+			[/\d[\d']*\d(@integersuffix)/, 'number'],
+			[/\d(@integersuffix)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/,  'comment.doc', '@doccomment' ],
+			[/\/\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],	// this breaks block comments in the shape of /* //*/
+			[/\*\//,    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		doccomment: [
+			[/[^\/*]+/, 'comment.doc' ],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/,    'comment.doc.invalid' ],
+			[/\*\//,    'comment.doc', '@pop'  ],
+			[/[\/*]/,   'comment.doc' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        'string', '@pop' ]
+		],
+	},
+};

src/csharp.ts

@@ -0,0 +1,175 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.cs',
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	keywords: [
+		'extern', 'alias', 'using', 'bool', 'decimal', 'sbyte', 'byte', 'short',
+		'ushort', 'int', 'uint', 'long', 'ulong', 'char', 'float', 'double',
+		'object', 'dynamic', 'string', 'assembly', 'is', 'as', 'ref',
+		'out', 'this', 'base', 'new', 'typeof', 'void', 'checked', 'unchecked',
+		'default', 'delegate', 'var', 'const', 'if', 'else', 'switch', 'case',
+		'while', 'do', 'for', 'foreach', 'in', 'break', 'continue', 'goto',
+		'return', 'throw', 'try', 'catch', 'finally', 'lock', 'yield', 'from',
+		'let', 'where', 'join', 'on', 'equals', 'into', 'orderby', 'ascending',
+		'descending', 'select', 'group', 'by', 'namespace', 'partial', 'class',
+		'field', 'event', 'method', 'param', 'property', 'public', 'protected',
+		'internal', 'private', 'abstract', 'sealed', 'static', 'struct', 'readonly',
+		'volatile', 'virtual', 'override', 'params', 'get', 'set', 'add', 'remove',
+		'operator', 'true', 'false', 'implicit', 'explicit', 'interface', 'enum',
+		'null', 'async', 'await','fixed','sizeof','stackalloc','unsafe', 'nameof',
+		'when'
+		],
+
+	namespaceFollows: [
+		'namespace', 'using',
+	],
+
+	parenFollows: [
+		'if', 'for', 'while', 'switch', 'foreach', 'using', 'catch', 'when'
+	],
+
+	operators: [
+		'=', '??', '||', '&&', '|', '^', '&', '==', '!=', '<=', '>=', '<<',
+		'+', '-', '*', '/', '%', '!', '~', '++', '--','+=',
+		'-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '>>', '=>'
+	],
+
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+
+	// escape sequences
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// identifiers and keywords
+			[/\@?[a-zA-Z_]\w*/, { cases: {
+				'@namespaceFollows': { token: 'keyword.$0', next: '@namespace' },
+				'@keywords': { token: 'keyword.$0', next: '@qualified' },
+				'@default': { token: 'identifier', next: '@qualified' }
+				}
+			}],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/}/, { cases: {
+					'$S2==interpolatedstring' : { token: 'string.quote', bracket: '@close', next: '@pop' }
+				,	'@default'   : '@brackets' } }],
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter', '@default'  : '' } } ],
+
+			// literal string
+			[/\@"/, { token: 'string.quote', bracket: '@open', next: '@litstring' } ],
+
+			// interpolated string
+			[/\$"/, { token: 'string.quote', bracket: '@open', next: '@interpolatedstring' } ],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?[fFdD]?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  { token: 'string.quote', bracket: '@open', next: '@string' } ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		qualified: [
+			[/[a-zA-Z_][\w]*/, { cases:
+				{ '@keywords': {token:'keyword.$0'},
+				'@default': 'identifier' }
+			}],
+			[/\./, 'delimiter'],
+			['','','@pop'],
+		],
+
+		namespace: [
+			{ include: '@whitespace' },
+			[/[A-Z]\w*/, 'namespace'],
+			[/[\.=]/, 'delimiter'],
+			['','','@pop'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/,    'comment', '@push' ],    // no nested comments :-(
+			['\\*/',    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		litstring: [
+			[/[^"]+/,    'string'],
+			[/""/,       'string.escape'],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		interpolatedstring: [
+			[/[^\\"{]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/{{/,       'string.escape'],
+			[/}}/,       'string.escape'],
+			[/{/,        { token: 'string.quote', bracket: '@open', next: 'root.interpolatedstring' } ],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		whitespace: [
+			[/^[ \t\v\f]*#\w.*$/, 'namespace.cpp' ],
+			[/[ \t\v\f\r\n]+/, ''],
+			[/\/\*/,       'comment', '@comment' ],
+			[/\/\/.*$/,    'comment'],
+		],
+	},
+};

src/dockerfile.ts

@@ -0,0 +1,97 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.dockerfile',
+
+	instructions: /FROM|MAINTAINER|RUN|EXPOSE|ENV|ADD|VOLUME|LABEL|USER|WORKDIR|COPY|CMD|ENTRYPOINT/,
+
+	instructionAfter: /ONBUILD/,
+
+	variableAfter: /ENV/,
+
+	variable:/\${?[\w]+}?/,
+
+	tokenizer: {
+	root: [
+			{ include: '@whitespace' },
+			{ include: '@comment' },
+
+			[/(@instructionAfter)(\s+)/, ['keyword', { token: '', next: '@instructions' }]],
+			['', 'keyword', '@instructions']
+	],
+
+	instructions: [
+			[/(@variableAfter)(\s+)([\w]+)/, ['keyword', '',{token:'variable', next:'@arguments'}]],
+			[/(@instructions)/, 'keyword', '@arguments']
+	],
+
+	arguments: [
+			{ include: '@whitespace' },
+			{ include: '@strings' },
+
+			[/(@variable)/, { cases: { '@eos': {token:'variable', next:'@popall'}, '@default': 'variable' }} ],
+			[/\\/, { cases: { '@eos': '', '@default': '' }}],
+			[/./, { cases: { '@eos': {token:'', next:'@popall'}, '@default': '' } }],
+	],
+
+	// Deal with white space, including comments
+	whitespace: [
+		[/\s+/, { cases: { '@eos': {token:'', next:'@popall'}, '@default': '' }}],
+	],
+
+	comment: [
+		[/(^#.*$)/, 'comment', '@popall']
+	],
+
+	// Recognize strings, including those broken across lines with \ (but not without)
+	strings: [
+		[/'$/, 'string', '@popall'],
+		[/'/, 'string', '@stringBody'],
+		[/"$/, 'string', '@popall'],
+		[/"/, 'string', '@dblStringBody']
+	],
+	stringBody: [
+		[/[^\\\$']/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+
+		[/\\./, 'string.escape'],
+		[/'$/, 'string', '@popall'],
+		[/'/, 'string', '@pop'],
+		[/(@variable)/, 'variable' ],
+
+		[/\\$/, 'string'],
+		[/$/, 'string', '@popall']
+	],
+	dblStringBody: [
+		[/[^\\\$"]/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+
+		[/\\./, 'string.escape'],
+		[/"$/, 'string', '@popall'],
+		[/"/, 'string', '@pop'],
+		[/(@variable)/, 'variable' ],
+
+		[/\\$/, 'string'],
+		[/$/, 'string', '@popall']
+	]
+	}
+};

src/fsharp.ts

@@ -0,0 +1,133 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['(*', '*)'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.fs',
+
+	keywords: [
+		'abstract', 'and', 'atomic', 'as',
+		'assert', 'asr', 'base', 'begin',
+		'break', 'checked', 'component',
+		'const', 'constraint', 'constructor',
+		'continue', 'class', 'default',
+		'delegate','do', 'done', 'downcast',
+		'downto', 'elif', 'else', 'end',
+		'exception', 'eager', 'event', 'external',
+		'extern',	'false', 'finally',	'for',
+		'fun',	'function', 'fixed', 'functor',
+		'global', 'if', 'in', 'include', 'inherit',
+		'inline', 'interface', 'internal', 'land',
+		'lor', 'lsl','lsr', 'lxor', 'lazy', 'let',
+		'match', 'member','mod','module', 'mutable',
+		'namespace', 'method', 'mixin', 'new', 'not',
+		'null', 'of', 'open', 'or', 'object',
+		'override', 'private', 'parallel', 'process',
+		'protected', 'pure', 'public', 'rec', 'return',
+		'static', 'sealed', 'struct', 'sig', 'then',
+		'to', 'true', 'tailcall', 'trait',
+		'try', 'type', 'upcast', 'use',
+		'val', 'void', 'virtual', 'volatile',
+		'when', 'while','with', 'yield'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\^%;\.,\/]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /[uU]?[yslnLI]?/,
+	floatsuffix: /[fFmM]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [< attributes >].
+			[/\[<.*>\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^#(if|else|endif)/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, 'delimiter' ],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/0x[0-9a-fA-F]+LF/, 'number.float'],
+			[/0x[0-9a-fA-F]+(@integersuffix)/, 'number.hex'],
+			[/0b[0-1]+(@integersuffix)/, 'number.bin'],
+			[/\d+(@integersuffix)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"""/, 'string', '@string."""'],
+			[/"/, 'string', '@string."' ],
+
+			// literal string
+			[/\@"/, { token: 'string.quote', bracket: '@open', next: '@litstring' }],
+
+			// characters
+			[/'[^\\']'B?/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\(\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\*]+/, 'comment' ],
+			[/\*\)/,    'comment', '@pop'  ],
+			[/\*/,   'comment' ]
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/("""|"B?)/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': 'string' }} ]
+		],
+
+		litstring: [
+			[/[^"]+/,    'string'],
+			[/""/,       'string.escape'],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+	},
+};

src/go.ts

@@ -0,0 +1,166 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	]
+};
+
+export var language = <ILanguage> {
+
+	defaultToken: '',
+	tokenPostfix: '.go',
+
+	keywords: [
+		'break',
+		'case',
+		'chan',
+		'const',
+		'continue',
+		'default',
+		'defer',
+		'else',
+		'fallthrough',
+		'for',
+		'func',
+		'go',
+		'goto',
+		'if',
+		'import',
+		'interface',
+		'map',
+		'package',
+		'range',
+		'return',
+		'select',
+		'struct',
+		'switch',
+		'type',
+		'var',
+		'bool',
+		'true',
+		'false',
+		'uint8',
+		'uint16',
+		'uint32',
+		'uint64',
+		'int8',
+		'int16',
+		'int32',
+		'int64',
+		'float32',
+		'float64',
+		'complex64',
+		'complex128',
+		'byte',
+		'rune',
+		'uint',
+		'int',
+		'uintptr',
+		'string',
+		'nil',
+	],
+
+	operators: [
+		'+', '-', '*', '/', '%', '&', '|', '^', '<<', '>>', '&^',
+		'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '&^=',
+		'&&', '||', '<-', '++', '--', '==', '<', '>', '=', '!', '!=', '<=', '>=', ':=', '...',
+		'(', ')', '', ']', '{', '}', ',', ';', '.', ':'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+																	'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [[ attributes ]].
+			[/\[\[.*\]\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+															'@default'  : '' } } ],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F']*[0-9a-fA-F]/, 'number.hex'],
+			[/0[0-7']*[0-7]/, 'number.octal'],
+			[/0[bB][0-1']*[0-1]/, 'number.binary'],
+			[/\d[\d']*/, 'number'],
+			[/\d/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/,  'comment.doc', '@doccomment' ],
+			[/\/\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//,    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		doccomment: [
+			[/[^\/*]+/, 'comment.doc' ],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/,    'comment.doc.invalid' ],
+			[/\*\//,    'comment.doc', '@pop'  ],
+			[/[\/*]/,   'comment.doc' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        'string', '@pop' ]
+		],
+	},
+};

src/ini.ts

@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.ini',
+
+	// we include these common regular expressions
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// sections
+			[/^\[[^\]]*\]/, 'metatag'],
+
+			// keys
+			[/(^\w+)(\s*)(\=)/, ['key', '', 'delimiter']],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// numbers
+			[/\d+/, 'number'],
+
+			// strings: recover on non-terminated strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/^\s*[#;].*$/,    		'comment'],
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+															'@default': 'string' }} ]
+		],
+	},
+};

src/jade.ts

@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.jade',
+
+	ignoreCase: true,
+
+	brackets: [
+			{ token:'delimiter.curly', open: '{', close: '}' },
+			{ token:'delimiter.array', open: '[', close: ']' },
+			{ token:'delimiter.parenthesis', open: '(', close: ')' }
+	],
+
+	keywords: [	'append', 'block', 'case', 'default', 'doctype', 'each', 'else', 'extends',
+		'for', 'if', 'in', 'include', 'mixin', 'typeof', 'unless', 'var', 'when'],
+
+	tags: [
+		'a', 'abbr', 'acronym', 'address', 'area', 'article', 'aside', 'audio',
+		'b', 'base', 'basefont', 'bdi', 'bdo', 'blockquote', 'body', 'br', 'button',
+		'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
+		'datalist', 'dd', 'del', 'details', 'dfn', 'div', 'dl', 'dt',
+		'em', 'embed',
+		'fieldset', 'figcaption', 'figure', 'font', 'footer', 'form', 'frame', 'frameset',
+		'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'header', 'hgroup', 'hr', 'html',
+		'i', 'iframe', 'img', 'input', 'ins',
+		'keygen', 'kbd',
+		'label', 'li', 'link',
+		'map', 'mark', 'menu', 'meta', 'meter',
+		'nav', 'noframes', 'noscript',
+		'object', 'ol', 'optgroup', 'option', 'output',
+		'p', 'param', 'pre', 'progress',
+		'q',
+		'rp', 'rt', 'ruby',
+		's', 'samp', 'script', 'section', 'select', 'small', 'source', 'span', 'strike', 'strong', 'style', 'sub', 'summary', 'sup',
+		'table', 'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'time', 'title', 'tr', 'tracks', 'tt',
+		'u', 'ul',
+		'video',
+		'wbr'
+	],
+
+	// we include these common regular expressions
+	symbols: /[\+\-\*\%\&\|\!\=\/\.\,\:]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	tokenizer: {
+		root: [
+
+			// Tag or a keyword at start
+			[/^(\s*)([a-zA-Z_-][\w-]*)/,
+				{ cases: {
+					'$2@tags': { cases: { '@eos': ['', 'tag'], '@default': ['', { token: 'tag', next: '@tag.$1' }, ] } },
+					'$2@keywords': [ '', { token: 'keyword.$2'}, ],
+					'@default': [ '', '', ]}}
+			],
+
+			// id
+			[/^(\s*)(#[a-zA-Z_-][\w-]*)/, { cases: { '@eos': ['', 'tag.id'], '@default': ['', { token: 'tag.id', next: '@tag.$1' }] }}],
+
+			// class
+			[/^(\s*)(\.[a-zA-Z_-][\w-]*)/, { cases: { '@eos': ['', 'tag.class'], '@default': ['', { token: 'tag.class', next: '@tag.$1' }] } }],
+
+			// plain text with pipe
+			[/^(\s*)(\|.*)$/, '' ],
+
+			{ include: '@whitespace' },
+
+			// keywords
+			[/[a-zA-Z_$][\w$]*/, { cases: { '@keywords': {token:'keyword.$0'},
+											'@default': '' } }],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d+\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/\d+/, 'number'],
+
+			// strings:
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		tag: [
+			[/(\.)(\s*$)/, [ {token: 'delimiter', next:'@blockText.$S2.'}, '']],
+			[/\s+/, { token: '', next: '@simpleText' }],
+
+			// id
+			[/#[a-zA-Z_-][\w-]*/, { cases: { '@eos': { token: 'tag.id', next: '@pop' }, '@default': 'tag.id' } }],
+			// class
+			[/\.[a-zA-Z_-][\w-]*/, { cases: { '@eos': { token: 'tag.class', next: '@pop' }, '@default': 'tag.class' } }],
+			// attributes
+			[/\(/, { token: 'delimiter.parenthesis', bracket: '@open', next: '@attributeList' }],
+		],
+
+		simpleText: [
+			[/[^#]+$/, {token: '', next: '@popall'}],
+			[/[^#]+/, {token: ''}],
+
+			// interpolation
+			[/(#{)([^}]*)(})/, { cases: {
+				'@eos': ['interpolation.delimiter', 'interpolation', { token: 'interpolation.delimiter', next: '@popall' }],
+				'@default': ['interpolation.delimiter', 'interpolation', 'interpolation.delimiter'] }}],
+
+			[/#$/, { token: '', next: '@popall' }],
+			[/#/, '']
+		],
+
+		attributeList: [
+			[/\s+/, '' ],
+			[/(\w+)(\s*=\s*)("|')/, ['attribute.name', 'delimiter', { token: 'attribute.value', next:'@value.$3'}]],
+			[/\w+/, 'attribute.name'],
+
+
+			[/,/, { cases: { '@eos': { token: 'attribute.delimiter', next: '@popall' }, '@default': 'attribute.delimiter' } }],
+
+			[/\)$/, { token: 'delimiter.parenthesis', bracket: '@close', next: '@popall' }],
+			[/\)/, { token: 'delimiter.parenthesis', bracket: '@close', next: '@pop' }],
+		],
+
+		whitespace: [
+			[/^(\s*)(\/\/.*)$/, { token: 'comment', next: '@blockText.$1.comment' } ],
+			[/[ \t\r\n]+/, ''],
+			[/<!--/, { token: 'comment', bracket: '@open', next: '@comment' }],
+		],
+
+		blockText: [
+			[/^\s+.*$/, { cases: { '($S2\\s+.*$)': { token: '$S3' }, '@default': { token: '@rematch', next: '@popall' } } }],
+			[/./,  { token: '@rematch', next: '@popall' }]
+		],
+
+		comment: [
+			[/[^<\-]+/, 'comment.content' ],
+			[/-->/,  { token: 'comment', bracket: '@close', next: '@pop' } ],
+			[/<!--/, 'comment.content.invalid'],
+			[/[<\-]/, 'comment.content' ]
+		],
+
+		string: [
+			[/[^\\"'#]+/, { cases: { '@eos': { token: 'string', next: '@popall' }, '@default': 'string' } }],
+			[/@escapes/, { cases: { '@eos': { token: 'string.escape', next: '@popall' }, '@default': 'string.escape' }}],
+			[/\\./, { cases: { '@eos': { token: 'string.escape.invalid', next: '@popall' }, '@default': 'string.escape.invalid' }}],
+			// interpolation
+			[/(#{)([^}]*)(})/, ['interpolation.delimiter', 'interpolation', 'interpolation.delimiter']],
+			[/#/, 'string'],
+			[/["']/, { cases: { '$#==$S2': { token: 'string', next: '@pop' }, '@default': { token: 'string' } } }],
+		],
+
+		// Almost identical to above, except for escapes and the output token
+		value: [
+			[/[^\\"']+/, { cases: { '@eos': { token: 'attribute.value', next: '@popall' }, '@default': 'attribute.value' }}],
+			[/\\./, { cases: { '@eos': { token: 'attribute.value', next: '@popall' }, '@default': 'attribute.value' }}],
+			[/["']/, { cases: { '$#==$S2': { token: 'attribute.value', next: '@pop' }, '@default': { token: 'attribute.value' } } }],
+		],
+	},
+};

src/java.ts

@@ -0,0 +1,130 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.java',
+
+	keywords: [
+		'abstract', 'continue', 'for', 'new', 'switch', 'assert', 'default',
+		'goto', 'package', 'synchronized', 'boolean', 'do', 'if', 'private',
+		'this', 'break', 'double', 'implements', 'protected', 'throw', 'byte',
+		'else', 'import', 'public', 'throws', 'case', 'enum', 'instanceof', 'return',
+		'transient', 'catch', 'extends', 'int', 'short', 'try', 'char', 'final',
+		'interface', 'static', 'void', 'class', 'finally', 'long', 'strictfp',
+		'volatile', 'const', 'float', 'native', 'super', 'while', 'true', 'false'
+	],
+
+	operators: [
+		'=', '>', '<', '!', '~', '?', ':',
+		'==', '<=', '>=', '!=', '&&', '||', '++', '--',
+		'+', '-', '*', '/', '&', '|', '^', '%', '<<',
+		'>>', '>>>', '+=', '-=', '*=', '/=', '&=', '|=',
+		'^=', '%=', '<<=', '>>=', '>>>='
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_$][\w$]*/, { cases: { '@keywords': {token:'keyword.$0'},
+																	'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+															'@default'  : '' } } ],
+
+			// @ annotations.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, 'annotation'],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/0[xX](@hexdigits)[Ll]?/, 'number.hex'],
+			[/0(@octaldigits)[Ll]?/, 'number.octal'],
+			[/0[bB](@binarydigits)[Ll]?/, 'number.binary'],
+			[/(@digits)[fFdD]/, 'number.float'],
+			[/(@digits)[lL]?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/,  'comment.doc', '@javadoc' ],
+			[/\/\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//,    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		javadoc: [
+			[/[^\/*]+/, 'comment.doc' ],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/,    'comment.doc.invalid' ],
+			[/\*\//,    'comment.doc', '@pop'  ],
+			[/[\/*]/,   'comment.doc' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        'string', '@pop' ]
+		],
+	},
+};

src/lua.ts

@@ -0,0 +1,107 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '--',
+		blockComment: ['--[[', ']]'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.lua',
+
+	keywords: [
+		'and', 'break', 'do', 'else', 'elseif',
+		'end', 'false', 'for', 'function', 'goto', 'if',
+		'in', 'local', 'nil', 'not', 'or',
+		'repeat', 'return', 'then', 'true', 'until',
+		'while'
+	],
+
+	brackets: [
+		{ token: 'delimiter.bracket', open: '{', close: '}'},
+		{ token: 'delimiter.array', open: '[', close: ']'},
+		{ token: 'delimiter.parenthesis', open: '(', close: ')'}
+	],
+
+	operators: [
+		'+', '-', '*', '/', '%', '^', '#', '==', '~=', '<=', '>=', '<', '>', '=',
+		';', ':', ',', '.', '..', '...'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+			// whitespace
+			{ include: '@whitespace' },
+
+		// keys
+		[/(,)(\s*)([a-zA-Z_]\w*)(\s*)(:)(?!:)/, ['delimiter', '', 'key', '', 'delimiter']],
+		[/({)(\s*)([a-zA-Z_]\w*)(\s*)(:)(?!:)/, ['@brackets', '', 'key', '', 'delimiter']],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+									'@default'  : '' } } ],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings: recover on non-terminated strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/--\[([=]*)\[/,  'comment', '@comment.$1' ],
+			[/--.*$/,        'comment'],
+		],
+
+		comment: [
+			[/[^\]]+/, 'comment'],
+			[/\]([=]*)\]/, { cases: { '$1==$S2': { token: 'comment', next: '@pop' }, '@default' : 'comment' } } ],
+			[/./,   'comment' ]
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': 'string' }} ]
+		],
+
+	},
+};

src/monaco.contribution.ts

@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+declare var require:<T>(moduleId:[string], callback:(module:T)=>void, error:(err:any)=>void)=>void;
+
+// Allow for running under nodejs/requirejs in tests
+var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+interface ILang extends monaco.languages.ILanguageExtensionPoint {
+	module: string;
+}
+
+interface ILangImpl {
+	conf: monaco.languages.IRichLanguageConfiguration;
+	language: monaco.languages.IMonarchLanguage;
+}
+
+let languageDefinitions:{[languageId:string]:ILang} = {};
+
+export function loadLanguage(languageId:string): monaco.Promise<void> {
+	let module = languageDefinitions[languageId].module;
+	return new _monaco.Promise<void>((c, e, p) => {
+		require<ILangImpl>([module], (mod) => {
+			_monaco.languages.setMonarchTokensProvider(languageId, mod.language);
+			_monaco.languages.setLanguageConfiguration(languageId, mod.conf);
+			c(void 0);
+		}, e);
+	});
+}
+
+function registerLanguage(def:ILang): void {
+	let languageId = def.id;
+
+	languageDefinitions[languageId] = def;
+	_monaco.languages.register(def);
+	_monaco.languages.onLanguage(languageId, () => {
+		loadLanguage(languageId);
+	});
+}
+
+
+registerLanguage({
+	id: 'bat',
+	extensions: [ '.bat', '.cmd'],
+	aliases: [ 'Batch', 'bat' ],
+	module: './bat'
+});
+registerLanguage({
+	id: 'coffeescript',
+	extensions: [ '.coffee' ],
+	aliases: [ 'CoffeeScript', 'coffeescript', 'coffee' ],
+	mimetypes: ['text/x-coffeescript', 'text/coffeescript'],
+	module: './coffee'
+});
+registerLanguage({
+	id: 'c',
+	extensions: [ '.c', '.h' ],
+	aliases: [ 'C', 'c' ],
+	module: './cpp'
+});
+registerLanguage({
+	id: 'cpp',
+	extensions: [ '.cpp', '.cc', '.cxx', '.hpp', '.hh', '.hxx' ],
+	aliases: [ 'C++', 'Cpp', 'cpp'],
+	module: './cpp'
+});
+registerLanguage({
+	id: 'csharp',
+	extensions: [ '.cs', '.csx' ],
+	aliases: [ 'C#', 'csharp' ],
+	module: './csharp'
+});
+registerLanguage({
+	id: 'dockerfile',
+	extensions: [ '.dockerfile' ],
+	filenames: [ 'Dockerfile' ],
+	aliases: [ 'Dockerfile' ],
+	module: './dockerfile'
+});
+registerLanguage({
+	id: 'fsharp',
+	extensions: [ '.fs', '.fsi', '.ml', '.mli', '.fsx', '.fsscript' ],
+	aliases: [ 'F#', 'FSharp', 'fsharp' ],
+	module: './fsharp'
+});
+registerLanguage({
+	id: 'go',
+	extensions: [ '.go' ],
+	aliases: [ 'Go' ],
+	module: './go'
+});
+registerLanguage({
+	id: 'ini',
+	extensions: [ '.ini', '.properties', '.gitconfig' ],
+	filenames: ['config', '.gitattributes', '.gitconfig', '.editorconfig'],
+	aliases: [ 'Ini', 'ini' ],
+	module: './ini'
+});
+registerLanguage({
+	id: 'jade',
+	extensions: [ '.jade', '.pug' ],
+	aliases: [ 'Jade', 'jade' ],
+	module: './jade'
+});
+registerLanguage({
+	id: 'java',
+	extensions: [ '.java', '.jav' ],
+	aliases: [ 'Java', 'java' ],
+	mimetypes: ['text/x-java-source', 'text/x-java'],
+	module: './java'
+});
+registerLanguage({
+	id: 'lua',
+	extensions: [ '.lua' ],
+	aliases: [ 'Lua', 'lua' ],
+	module: './lua'
+});
+registerLanguage({
+	id: 'objective-c',
+	extensions: [ '.m' ],
+	aliases: [ 'Objective-C'],
+	module: './objective-c'
+});
+registerLanguage({
+	id: 'powershell',
+	extensions: [ '.ps1', '.psm1', '.psd1' ],
+	aliases: [ 'PowerShell', 'powershell', 'ps', 'ps1' ],
+	module: './powershell'
+});
+registerLanguage({
+	id: 'python',
+	extensions: [ '.py', '.rpy', '.pyw', '.cpy', '.gyp', '.gypi' ],
+	aliases: [ 'Python', 'py' ],
+	firstLine: '^#!/.*\\bpython[0-9.-]*\\b',
+	module: './python'
+});
+registerLanguage({
+	id: 'r',
+	extensions: [ '.r', '.rhistory', '.rprofile', '.rt' ],
+	aliases: [ 'R', 'r' ],
+	module: './r'
+});
+registerLanguage({
+	id: 'ruby',
+	extensions: [ '.rb', '.rbx', '.rjs', '.gemspec', '.pp' ],
+	filenames: [ 'rakefile' ],
+	aliases: [ 'Ruby', 'rb' ],
+	module: './ruby'
+});
+registerLanguage({
+	id: 'swift',
+	aliases: ['Swift','swift'],
+	extensions: ['.swift'],
+	mimetypes: ['text/swift'],
+	module: './swift'
+});
+registerLanguage({
+	id: 'sql',
+	extensions: [ '.sql' ],
+	aliases: [ 'SQL' ],
+	module: './sql'
+});
+registerLanguage({
+	id: 'vb',
+	extensions: [ '.vb' ],
+	aliases: [ 'Visual Basic', 'vb' ],
+	module: './vb'
+});
+registerLanguage({
+	id: 'xml',
+	extensions: [ '.xml', '.dtd', '.ascx', '.csproj', '.config', '.wxi', '.wxl', '.wxs', '.xaml', '.svg', '.svgz' ],
+	firstLine : '(\\<\\?xml.*)|(\\<svg)|(\\<\\!doctype\\s+svg)',
+	aliases: [ 'XML', 'xml' ],
+	mimetypes: ['text/xml', 'application/xml', 'application/xaml+xml', 'application/xml-dtd'],
+	module: './xml'
+});

src/objective-c.ts

@@ -0,0 +1,201 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.objective-c',
+
+	keywords: [
+		'#import',
+		'#include',
+		'#define',
+		'#else',
+		'#endif',
+		'#if',
+		'#ifdef',
+		'#ifndef',
+		'#ident',
+		'#undef',
+
+		'@class',
+		'@defs',
+		'@dynamic',
+		'@encode',
+		'@end',
+		'@implementation',
+		'@interface',
+		'@package',
+		'@private',
+		'@protected',
+		'@property',
+		'@protocol',
+		'@public',
+		'@selector',
+		'@synthesize',
+
+		'__declspec',
+
+		'assign',
+		'auto',
+
+		'BOOL',
+		'break',
+		'bycopy',
+		'byref',
+
+		'case',
+		'char',
+		'Class',
+		'const',
+		'copy',
+		'continue',
+
+		'default',
+		'do',
+		'double',
+
+		'else',
+		'enum',
+		'extern',
+
+		'FALSE',
+		'false',
+		'float',
+		'for',
+
+		'goto',
+
+		'if',
+		'in',
+		'int',
+		'id',
+		'inout',
+		'IMP',
+
+		'long',
+
+		'nil',
+		'nonatomic',
+		'NULL',
+
+		'oneway',
+		'out',
+
+		'private',
+		'public',
+		'protected',
+
+		'readwrite',
+		'readonly',
+		'register',
+		'return',
+
+		'SEL',
+		'self',
+		'short',
+		'signed',
+		'sizeof',
+		'static',
+		'struct',
+		'super',
+		'switch',
+
+		'typedef',
+		'TRUE',
+		'true',
+
+		'union',
+		'unsigned',
+
+		'volatile',
+		'void',
+
+		'while',
+	],
+
+	decpart: /\d(_?\d)*/,
+	decimal: /0|@decpart/,
+
+	tokenizer: {
+	root: [
+			{ include: '@comments' },
+			{ include: '@whitespace' },
+			{ include: '@numbers' },
+			{ include: '@strings' },
+
+			[/[,:;]/, 'delimiter'],
+			[/[{}\[\]()<>]/, '@brackets'],
+
+			[/[a-zA-Z@#]\w*/, { cases: { '@keywords': 'keyword',
+										'@default': 'identifier' } }],
+
+			[/[<>=\\+\\-\\*\\/\\^\\|\\~,]|and\\b|or\\b|not\\b]/, 'operator'],
+	],
+
+	whitespace: [
+		[/\s+/, 'white'],
+	],
+
+	comments: [
+			['\\/\\*','comment','@comment' ],
+			['\\/\\/+.*','comment' ],
+		],
+
+	comment: [
+			['\\*\\/','comment','@pop' ],
+			['.', 'comment', ],
+		],
+
+	numbers: [
+			[/0[xX][0-9a-fA-F]*(_?[0-9a-fA-F])*/, 'number.hex'],
+			[/@decimal((\.@decpart)?([eE][\-+]?@decpart)?)[fF]*/, {
+				cases: {  	'(\\d)*': 'number',
+							'$0':'number.float' }} ]
+		],
+
+	// Recognize strings, including those broken across lines with \ (but not without)
+	strings: [
+		[/'$/, 'string.escape', '@popall'],
+		[/'/, 'string.escape', '@stringBody'],
+		[/"$/, 'string.escape', '@popall'],
+		[/"/, 'string.escape', '@dblStringBody']
+	],
+	stringBody: [
+		[/\\./, 'string'],
+		[/'/, 'string.escape', '@popall'],
+		[/.(?=.*')/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	],
+	dblStringBody: [
+		[/\\./, 'string'],
+		[/"/, 'string.escape', '@popall'],
+		[/.(?=.*")/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	]
+	}
+};

src/powershell.ts

@@ -0,0 +1,125 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	// the default separators except `$-`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '#',
+		blockComment: ['<#', '#>'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [
+	// 			{ tokenType:'string', openTrigger: '"', open: /@"$/, closeComplete: '"@' },
+	// 			{ tokenType:'string', openTrigger: '\'', open: /@'$/, closeComplete: '\'@' },
+	// 			{ tokenType:'string', openTrigger: '"', open: /"$/, closeComplete: '"' },
+	// 			{ tokenType: 'string', openTrigger: '\'', open: /'$/, closeComplete: '\'' }
+	// ],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.ps1',
+
+	brackets: [
+		{ token: 'delimiter.curly', open: '{', close: '}' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' }
+	],
+
+	keywords: [
+		'begin', 'break', 'catch', 'class', 'continue', 'data',
+		'define', 'do', 'dynamicparam', 'else', 'elseif', 'end',
+		'exit', 'filter', 'finally', 'for', 'foreach', 'from',
+		'function', 'if', 'in', 'param', 'process', 'return',
+		'switch', 'throw', 'trap', 'try', 'until', 'using',
+		'var', 'while', 'workflow', 'parallel', 'sequence', 'inlinescript', 'configuration'
+	],
+
+	helpKeywords: /SYNOPSIS|DESCRIPTION|PARAMETER|EXAMPLE|INPUTS|OUTPUTS|NOTES|LINK|COMPONENT|ROLE|FUNCTIONALITY|FORWARDHELPTARGETNAME|FORWARDHELPCATEGORY|REMOTEHELPRUNSPACE|EXTERNALHELP/,
+
+	// we include these common regular expressions
+	symbols: /[=><!~?&%|+\-*\/\^;\.,]+/,
+	escapes: /`(?:[abfnrtv\\"'$]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// commands and keywords
+			[/[a-zA-Z_][\w-]*/, { cases: { '@keywords': {token:'keyword.$0'},
+											'@default': '' } }],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// labels
+			[/^:\w*/, 'metatag'],
+
+			// variables
+			[/\$(\{((global|local|private|script|using):)?[\w]+\}|((global|local|private|script|using):)?[\w]+)/, 'variable'],
+
+			// Comments
+			[/<#/, 'comment', '@comment'],
+			[/#.*$/, 'comment'],
+
+			// delimiters
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings:
+			[/\@"/, 'string', '@herestring."'],
+			[/\@'/,  'string', '@herestring.\''],
+			[/"/,  { cases: { '@eos': 'string', '@default': {token:'string', next:'@string."'} }} ],
+			[/'/, { cases: { '@eos': 'string', '@default': {token:'string', next:'@string.\''} }} ],
+		],
+
+		string: [
+			[/[^"'\$`]+/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+			[/@escapes/, { cases: { '@eos': {token:'string.escape', next:'@popall'}, '@default': 'string.escape' }}],
+			[/`./, { cases: { '@eos': {token:'string.escape.invalid', next:'@popall'}, '@default': 'string.escape.invalid' }}],
+
+			[/\$[\w]+$/, { cases: { '$S2=="': { token: 'variable', next: '@popall' }, '@default': { token: 'string', next: '@popall' } } }],
+			[/\$[\w]+/, { cases: { '$S2=="': 'variable', '@default': 'string' }}],
+
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }} }} ],
+		],
+
+		herestring: [
+			[/^\s*(["'])@/, { cases: { '$1==$S2': { token: 'string', next: '@pop' }, '@default': 'string' } }],
+			[/[^\$`]+/,'string' ],
+			[/@escapes/, 'string.escape'],
+			[/`./, 'string.escape.invalid'],
+			[/\$[\w]+/, { cases: { '$S2=="': 'variable', '@default': 'string' } }],
+		],
+
+		comment: [
+			[/[^#\.]+/, 'comment' ],
+			[/#>/, 'comment', '@pop'],
+			[/(\.)(@helpKeywords)(?!\w)/, { token: 'comment.keyword.$2' } ],
+			[/[\.#]/,   'comment' ]
+		],
+	},
+};

src/python.ts

@@ -0,0 +1,232 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+		blockComment: ['\'\'\'', '\'\'\''],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// Cause an automatic indent to occur after lines ending in :.
+	// enhancedBrackets: [ { open: /.*:\s*$/,  closeComplete: 'else:' } ],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.python',
+
+	keywords: [
+		'and',
+		'as',
+		'assert',
+		'break',
+		'class',
+		'continue',
+		'def',
+		'del',
+		'elif',
+		'else',
+		'except',
+		'exec',
+		'finally',
+		'for',
+		'from',
+		'global',
+		'if',
+		'import',
+		'in',
+		'is',
+		'lambda',
+		'None',
+		'not',
+		'or',
+		'pass',
+		'print',
+		'raise',
+		'return',
+		'self',
+		'try',
+		'while',
+		'with',
+		'yield',
+
+		'int',
+		'float',
+		'long',
+		'complex',
+		'hex',
+
+		'abs',
+		'all',
+		'any',
+		'apply',
+		'basestring',
+		'bin',
+		'bool',
+		'buffer',
+		'bytearray',
+		'callable',
+		'chr',
+		'classmethod',
+		'cmp',
+		'coerce',
+		'compile',
+		'complex',
+		'delattr',
+		'dict',
+		'dir',
+		'divmod',
+		'enumerate',
+		'eval',
+		'execfile',
+		'file',
+		'filter',
+		'format',
+		'frozenset',
+		'getattr',
+		'globals',
+		'hasattr',
+		'hash',
+		'help',
+		'id',
+		'input',
+		'intern',
+		'isinstance',
+		'issubclass',
+		'iter',
+		'len',
+		'locals',
+		'list',
+		'map',
+		'max',
+		'memoryview',
+		'min',
+		'next',
+		'object',
+		'oct',
+		'open',
+		'ord',
+		'pow',
+		'print',
+		'property',
+		'reversed',
+		'range',
+		'raw_input',
+		'reduce',
+		'reload',
+		'repr',
+		'reversed',
+		'round',
+		'set',
+		'setattr',
+		'slice',
+		'sorted',
+		'staticmethod',
+		'str',
+		'sum',
+		'super',
+		'tuple',
+		'type',
+		'unichr',
+		'unicode',
+		'vars',
+		'xrange',
+		'zip',
+
+		'True',
+		'False',
+
+		'__dict__',
+		'__methods__',
+		'__members__',
+		'__class__',
+		'__bases__',
+		'__name__',
+		'__mro__',
+		'__subclasses__',
+		'__init__',
+		'__import__'
+	],
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.bracket' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	tokenizer: {
+	root: [
+			{ include: '@whitespace' },
+			{ include: '@numbers' },
+			{ include: '@strings' },
+
+			[/[,:;]/, 'delimiter'],
+			[/[{}\[\]()]/, '@brackets'],
+
+			[/@[a-zA-Z]\w*/, 'tag'],
+			[/[a-zA-Z]\w*/, { cases: { '@keywords': 'keyword',
+										'@default': 'identifier' } }]
+	],
+
+	// Deal with white space, including single and multi-line comments
+	whitespace: [
+		[/\s+/, 'white'],
+		[/(^#.*$)/, 'comment'],
+		[/('''.*''')|(""".*""")/, 'string'],
+		[/'''.*$/, 'string', '@endDocString'],
+		[/""".*$/, 'string', '@endDblDocString']
+	],
+	endDocString: [
+		[/\\'/, 'string'],
+		[/.*'''/, 'string', '@popall'],
+		[/.*$/, 'string']
+	],
+	endDblDocString: [
+		[/\\"/, 'string'],
+		[/.*"""/, 'string', '@popall'],
+		[/.*$/, 'string']
+	],
+
+	// Recognize hex, negatives, decimals, imaginaries, longs, and scientific notation
+	numbers: [
+		[/-?0x([abcdef]|[ABCDEF]|\d)+[lL]?/, 'number.hex'],
+		[/-?(\d*\.)?\d+([eE][+\-]?\d+)?[jJ]?[lL]?/, 'number']
+	],
+
+	// Recognize strings, including those broken across lines with \ (but not without)
+	strings: [
+		[/'$/, 'string.escape', '@popall'],
+		[/'/, 'string.escape', '@stringBody'],
+		[/"$/, 'string.escape', '@popall'],
+		[/"/, 'string.escape', '@dblStringBody']
+	],
+	stringBody: [
+		[/\\./, 'string'],
+		[/'/, 'string.escape', '@popall'],
+		[/.(?=.*')/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	],
+	dblStringBody: [
+		[/\\./, 'string'],
+		[/"/, 'string.escape', '@popall'],
+		[/.(?=.*")/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	]
+	}
+};

src/r.ts

@@ -0,0 +1,230 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.r',
+
+	roxygen: [
+		'@param',
+		'@return',
+		'@name',
+		'@rdname',
+		'@examples',
+		'@include',
+		'@docType',
+		'@S3method',
+		'@TODO',
+		'@aliases',
+		'@alias',
+		'@assignee',
+		'@author',
+		'@callGraphDepth',
+		'@callGraph',
+		'@callGraphPrimitives',
+		'@concept',
+		'@exportClass',
+		'@exportMethod',
+		'@exportPattern',
+		'@export',
+		'@formals',
+		'@format',
+		'@importClassesFrom',
+		'@importFrom',
+		'@importMethodsFrom',
+		'@import',
+		'@keywords',
+		'@method',
+		'@nord',
+		'@note',
+		'@references',
+		'@seealso',
+		'@setClass',
+		'@slot',
+		'@source',
+		'@title',
+		'@usage'
+	],
+
+	constants: [
+		'NULL',
+		'FALSE',
+		'TRUE',
+		'NA',
+		'Inf',
+		'NaN ',
+		'NA_integer_',
+		'NA_real_',
+		'NA_complex_',
+		'NA_character_ ',
+		'T',
+		'F',
+		'LETTERS',
+		'letters',
+		'month.abb',
+		'month.name',
+		'pi',
+		'R.version.string'
+	],
+
+	keywords: [
+		'break',
+		'next',
+		'return',
+		'if',
+		'else',
+		'for',
+		'in',
+		'repeat',
+		'while',
+		'array',
+		'category',
+		'character',
+		'complex',
+		'double',
+		'function',
+		'integer',
+		'list',
+		'logical',
+		'matrix',
+		'numeric',
+		'vector',
+		'data.frame',
+		'factor',
+		'library',
+		'require',
+		'attach',
+		'detach',
+		'source'
+	],
+
+	special: [
+		'\\n',
+		'\\r',
+		'\\t',
+		'\\b',
+		'\\a',
+		'\\f',
+		'\\v',
+		'\\\'',
+		'\\"',
+		'\\\\'
+	],
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.bracket' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	tokenizer: {
+		root: [
+			{ include: '@numbers' },
+			{ include: '@strings' },
+
+			[/[{}\[\]()]/, '@brackets'],
+
+			{ include: '@operators' },
+
+			[/#'/, 'comment.doc', '@roxygen'],
+			[/(^#.*$)/, 'comment'],
+
+			[/\s+/, 'white'],
+
+			[/[,:;]/, 'delimiter'],
+
+			[/@[a-zA-Z]\w*/, 'tag'],
+			[/[a-zA-Z]\w*/, {
+				cases: {
+					'@keywords': 'keyword',
+					'@constants': 'constant',
+					'@default': 'identifier'
+				}
+			}]
+		],
+
+		// Recognize Roxygen comments
+		roxygen: [
+			[/@\w+/, {
+				cases: {
+					'@roxygen': 'tag',
+					'@eos': { token: 'comment.doc', next: '@pop' },
+					'@default': 'comment.doc'
+				}
+			}],
+			[/\s+/, {
+				cases: {
+					'@eos': { token: 'comment.doc', next: '@pop' },
+					'@default': 'comment.doc'
+				}
+			}],
+			[/.*/, { token: 'comment.doc', next: '@pop' }]
+		],
+
+		// Recognize positives, negatives, decimals, imaginaries, and scientific notation
+		numbers: [
+			[/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number']
+		],
+
+		// Recognize operators
+		operators: [
+			[/<{1,2}-/, 'operator'],
+			[/->{1,2}/, 'operator'],
+			[/%[^%\s]+%/, 'operator'],
+			[/\*\*/, 'operator'],
+			[/%%/, 'operator'],
+			[/&&/, 'operator'],
+			[/\|\|/, 'operator'],
+			[/<</, 'operator'],
+			[/>>/, 'operator'],
+			[/[-+=&|!<>^~*/:$]/, 'operator']
+		],
+
+		// Recognize strings, including those broken across lines
+		strings: [
+			[/'/, 'string.escape', '@stringBody'],
+			[/"/, 'string.escape', '@dblStringBody']
+		],
+		stringBody: [
+			[/\\./, {
+				cases: {
+					'@special': 'string',
+					'@default': 'error-token'
+				}
+			}],
+			[/'/, 'string.escape', '@popall'],
+			[/./, 'string'],
+		],
+		dblStringBody: [
+			[/\\./, {
+				cases: {
+					'@special': 'string',
+					'@default': 'error-token'
+				}
+			}],
+			[/"/, 'string.escape', '@popall'],
+			[/./, 'string'],
+		]
+	}
+};

src/ruby.ts

@@ -0,0 +1,395 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+		blockComment: ['=begin', '=end'],
+	},
+	brackets: [['(',')'],['{','}'], ['[',']']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+	],
+	__electricCharacterSupport: {
+		// trigger outdenting on 'end'
+		embeddedElectricCharacters: ['d']
+	}
+};
+
+/*
+ * Ruby language definition
+ *
+ * Quite a complex language due to elaborate escape sequences
+ * and quoting of literate strings/regular expressions, and
+ * an 'end' keyword that does not always apply to modifiers like until and while,
+ * and a 'do' keyword that sometimes starts a block, but sometimes is part of
+ * another statement (like 'while').
+ *
+ * (1) end blocks:
+ * 'end' may end declarations like if or until, but sometimes 'if' or 'until'
+ * are modifiers where there is no 'end'. Also, 'do' sometimes starts a block
+ * that is ended by 'end', but sometimes it is part of a 'while', 'for', or 'until'
+ * To do proper brace matching we do some elaborate state manipulation.
+ * some examples:
+ *
+ *   until bla do
+ *     work until tired
+ *     list.each do
+ *       something if test
+ *     end
+ *   end
+ *
+ * or
+ *
+ * if test
+ *  something (if test then x end)
+ *  bar if bla
+ * end
+ *
+ * or, how about using class as a property..
+ *
+ * class Test
+ *   def endpoint
+ *     self.class.endpoint || routes
+ *   end
+ * end
+ *
+ * (2) quoting:
+ * there are many kinds of strings and escape sequences. But also, one can
+ * start many string-like things as '%qx' where q specifies the kind of string
+ * (like a command, escape expanded, regular expression, symbol etc.), and x is
+ * some character and only another 'x' ends the sequence. Except for brackets
+ * where the closing bracket ends the sequence.. and except for a nested bracket
+ * inside the string like entity. Also, such strings can contain interpolated
+ * ruby expressions again (and span multiple lines). Moreover, expanded
+ * regular expression can also contain comments.
+ */
+
+export var language = <ILanguage> {
+	tokenPostfix: '.ruby',
+
+	keywords: [
+		'__LINE__', '__ENCODING__', '__FILE__', 'BEGIN', 'END', 'alias', 'and', 'begin',
+		'break', 'case', 'class', 'def', 'defined?', 'do', 'else', 'elsif', 'end',
+		'ensure', 'for', 'false', 'if', 'in', 'module', 'next', 'nil', 'not', 'or', 'redo',
+		'rescue', 'retry', 'return', 'self', 'super', 'then', 'true', 'undef', 'unless',
+		'until', 'when', 'while', 'yield',
+	],
+
+	keywordops: [
+		'::', '..', '...', '?', ':', '=>'
+	],
+
+	builtins: [
+		'require', 'public', 'private', 'include', 'extend', 'attr_reader',
+		'protected', 'private_class_method', 'protected_class_method', 'new'
+	],
+
+	// these are closed by 'end' (if, while and until are handled separately)
+	declarations: [
+		'module','class','def','case','do','begin','for','if','while','until','unless'
+	],
+
+	linedecls: [
+		'def','case','do','begin','for','if','while','until','unless'
+	],
+
+	operators: [
+		'^', '&', '|', '<=>', '==', '===', '!~', '=~', '>', '>=', '<', '<=', '<<', '>>', '+',
+		'-', '*', '/', '%', '**', '~', '+@', '-@', '[]', '[]=', '`',
+		'+=', '-=', '*=', '**=', '/=', '^=', '%=', '<<=', '>>=', '&=', '&&=', '||=', '|='
+	],
+
+	brackets: [
+		{ open: '(', close: ')', token: 'delimiter.parenthesis'},
+		{ open: '{', close: '}', token: 'delimiter.curly'},
+		{ open: '[', close: ']', token: 'delimiter.square'}
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%\.]+/,
+
+	// escape sequences
+	escape:  /(?:[abefnrstv\\"'\n\r]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2}|u[0-9A-Fa-f]{4})/,
+	escapes: /\\(?:C\-(@escape|.)|c(@escape|.)|@escape)/,
+
+	decpart: /\d(_?\d)*/,
+	decimal: /0|@decpart/,
+
+	delim:     /[^a-zA-Z0-9\s\n\r]/,
+	heredelim: /(?:\w+|'[^']*'|"[^"]*"|`[^`]*`)/,
+
+	regexpctl: /[(){}\[\]\$\^|\-*+?\.]/,
+	regexpesc: /\\(?:[AzZbBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})?/,
+
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		// Main entry.
+		// root.<decl> where decl is the current opening declaration (like 'class')
+		root: [
+			// identifiers and keywords
+			// most complexity here is due to matching 'end' correctly with declarations.
+			// We distinguish a declaration that comes first on a line, versus declarations further on a line (which are most likey modifiers)
+			[/^(\s*)([a-z_]\w*[!?=]?)/, ['white',
+				{ cases: { 'for|until|while': { token: 'keyword.$2', bracket: '@open', next: '@dodecl.$2' },
+							'@declarations':   { token: 'keyword.$2', bracket: '@open', next: '@root.$2' },
+							'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' },
+							'@keywords': 'keyword',
+							'@builtins': 'predefined',
+							'@default': 'identifier' } }]],
+			[/[a-z_]\w*[!?=]?/,
+				{ cases: { 'if|unless|while|until': { token: 'keyword.$0x', bracket: '@open', next: '@modifier.$0x' },
+							'for': { token: 'keyword.$2', bracket: '@open', next: '@dodecl.$2' },
+							'@linedecls': { token: 'keyword.$0', bracket: '@open', next: '@root.$0' },
+							'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' },
+							'@keywords': 'keyword',
+							'@builtins': 'predefined',
+							'@default': 'identifier' } }],
+
+			[/[A-Z][\w]*[!?=]?/, 'constructor.identifier' ],     // constant
+			[/\$[\w]*/,    'global.constant' ],               // global
+			[/@[\w]*/,     'namespace.instance.identifier' ], // instance
+			[/@@[\w]*/,    'namespace.class.identifier' ],    // class
+
+			// here document
+			[/<<-(@heredelim).*/, { token: 'string.heredoc.delimiter', bracket: '@open', next: '@heredoc.$1' } ],
+			[/[ \t\r\n]+<<(@heredelim).*/, { token: 'string.heredoc.delimiter', bracket: '@open', next: '@heredoc.$1' } ],
+			[/^<<(@heredelim).*/, { token: 'string.heredoc.delimiter', bracket: '@open', next: '@heredoc.$1' } ],
+
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// strings
+			[/"/,  { token: 'string.d.delim', bracket: '@open', next: '@dstring.d."'} ],
+			[/'/,  { token: 'string.sq.delim', bracket: '@open', next: '@sstring.sq' } ],
+
+			// % literals. For efficiency, rematch in the 'pstring' state
+			[/%([rsqxwW]|Q?)/,  { token: '@rematch', next: 'pstring' } ],
+
+			// commands and symbols
+			[/`/,  { token: 'string.x.delim', bracket: '@open', next: '@dstring.x.`' } ],
+			[/:(\w|[$@])\w*[!?=]?/, 'string.s'],
+			[/:"/, { token: 'string.s.delim', bracket: '@open', next: '@dstring.s."' } ],
+			[/:'/, { token: 'string.s.delim', bracket: '@open', next: '@sstring.s' } ],
+
+			// regular expressions. Lookahead for a (not escaped) closing forwardslash on the same line
+			[/\/(?=(\\\/|[^\/\n])+\/)/, { token: 'regexp.delim', bracket: '@open', next: '@regexp' } ],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, { cases: { '@keywordops': 'keyword',
+									'@operators' : 'operator',
+									'@default'   : '' } } ],
+
+			[/[;,]/, 'delimiter'],
+
+			// numbers
+			[/0[xX][0-9a-fA-F](_?[0-9a-fA-F])*/, 'number.hex'],
+			[/0[_oO][0-7](_?[0-7])*/, 'number.octal'],
+			[/0[bB][01](_?[01])*/, 'number.binary'],
+			[/0[dD]@decpart/, 'number'],
+			[/@decimal((\.@decpart)?([eE][\-+]?@decpart)?)/, { cases: { '$1': 'number.float',
+																		'@default': 'number' }}],
+
+		],
+
+		// used to not treat a 'do' as a block opener if it occurs on the same
+		// line as a 'do' statement: 'while|until|for'
+		// dodecl.<decl> where decl is the declarations started, like 'while'
+		dodecl: [
+			[/^/, { token: '', switchTo: '@root.$S2' }], // get out of do-skipping mode on a new line
+			[/[a-z_]\w*[!?=]?/, { cases: { 'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' }, // end on same line
+											'do' : { token: 'keyword', switchTo: '@root.$S2' }, // do on same line: not an open bracket here
+											'@linedecls': { token: '@rematch', switchTo: '@root.$S2' }, // other declaration on same line: rematch
+											'@keywords': 'keyword',
+											'@builtins': 'predefined',
+											'@default': 'identifier' } }],
+			{ include: '@root' }
+		],
+
+		// used to prevent potential modifiers ('if|until|while|unless') to match
+		// with 'end' keywords.
+		// modifier.<decl>x where decl is the declaration starter, like 'if'
+		modifier: [
+			[/^/, '', '@pop'], // it was a modifier: get out of modifier mode on a new line
+			[/[a-z_]\w*[!?=]?/, { cases: { 'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' }, // end on same line
+											'then|else|elsif|do': { token: 'keyword', switchTo: '@root.$S2' }, // real declaration and not a modifier
+											'@linedecls': { token: '@rematch', switchTo: '@root.$S2' }, // other declaration => not a modifier
+											'@keywords': 'keyword',
+											'@builtins': 'predefined',
+											'@default': 'identifier' } }],
+			{ include: '@root' }
+		],
+
+		// single quote strings (also used for symbols)
+		// sstring.<kind>  where kind is 'sq' (single quote) or 's' (symbol)
+		sstring: [
+			[/[^\\']+/,      'string.$S2' ],
+			[/\\\\|\\'|\\$/, 'string.$S2.escape'],
+			[/\\./,          'string.$S2.invalid'],
+			[/'/,            { token: 'string.$S2.delim', bracket: '@close', next: '@pop'} ]
+		],
+
+		// double quoted "string".
+		// dstring.<kind>.<delim> where kind is 'd' (double quoted), 'x' (command), or 's' (symbol)
+		// and delim is the ending delimiter (" or `)
+		dstring: [
+			[/[^\\`"#]+/, 'string.$S2'],
+			[/#/,         'string.$S2.escape', '@interpolated' ],
+			[/\\$/,       'string.$S2.escape' ],
+			[/@escapes/,  'string.$S2.escape'],
+			[/\\./,       'string.$S2.escape.invalid'],
+			[/[`"]/,      { cases: { '$#==$S3':  { token: 'string.$S2.delim', bracket: '@close', next: '@pop'},
+										'@default': 'string.$S2' } } ]
+		],
+
+		// literal documents
+		// heredoc.<close> where close is the closing delimiter
+		heredoc: [
+			[/^(\s*)(@heredelim)$/, { cases: { '$2==$S2': ['string.heredoc', { token: 'string.heredoc.delimiter', bracket: '@close', next: '@pop' }],
+												'@default': ['string.heredoc','string.heredoc'] }}],
+			[/.*/, 'string.heredoc' ],
+		],
+
+		// interpolated sequence
+		interpolated: [
+			[/\$\w*/,      'global.constant', '@pop' ],
+			[/@\w*/,       'namespace.class.identifier', '@pop' ],
+			[/@@\w*/,      'namespace.instance.identifier', '@pop' ],
+			[/[{]/, { token: 'string.escape.curly', bracket: '@open', switchTo: '@interpolated_compound' }],
+			['', '', '@pop' ], // just a # is interpreted as a #
+		],
+
+		// any code
+		interpolated_compound: [
+			[/[}]/, { token: 'string.escape.curly', bracket: '@close', next: '@pop'} ],
+			{ include: '@root' },
+		],
+
+		// %r quoted regexp
+		// pregexp.<open>.<close> where open/close are the open/close delimiter
+		pregexp: [
+			{ include: '@whitespace' },
+			// turns out that you can quote using regex control characters, aargh!
+			// for example; %r|kgjgaj| is ok (even though | is used for alternation)
+			// so, we need to match those first
+			[/[^\(\{\[\\]/, { cases: { '$#==$S3' : { token: 'regexp.delim', bracket: '@close', next: '@pop' },
+										'$#==$S2' : { token: 'regexp.delim', bracket: '@open', next: '@push' }, // nested delimiters are allowed..
+										'~[)}\\]]' : '@brackets.regexp.escape.control',
+										'~@regexpctl': 'regexp.escape.control',
+										'@default': 'regexp' }}],
+			{ include: '@regexcontrol' },
+		],
+
+		// We match regular expression quite precisely
+		regexp: [
+			{ include:   '@regexcontrol' },
+			[/[^\\\/]/,  'regexp' ],
+			['/[ixmp]*', { token: 'regexp.delim', bracket: '@close'}, '@pop' ],
+		],
+
+		regexcontrol: [
+			[/(\{)(\d+(?:,\d*)?)(\})/, ['@brackets.regexp.escape.control', 'regexp.escape.control', '@brackets.regexp.escape.control'] ],
+			[/(\[)(\^?)/,     ['@brackets.regexp.escape.control',{ token: 'regexp.escape.control', next: '@regexrange'}]],
+			[/(\()(\?[:=!])/, ['@brackets.regexp.escape.control', 'regexp.escape.control'] ],
+			[/\(\?#/,         { token: 'regexp.escape.control', bracket: '@open', next: '@regexpcomment' }],
+			[/[()]/,        '@brackets.regexp.escape.control'],
+			[/@regexpctl/,  'regexp.escape.control'],
+			[/\\$/,         'regexp.escape' ],
+			[/@regexpesc/,  'regexp.escape' ],
+			[/\\\./,        'regexp.invalid' ],
+			[/#/,           'regexp.escape', '@interpolated' ],
+		],
+
+		regexrange: [
+			[/-/,     'regexp.escape.control'],
+			[/\^/,    'regexp.invalid'],
+			[/\\$/,   'regexp.escape' ],
+			[/@regexpesc/, 'regexp.escape'],
+			[/[^\]]/, 'regexp'],
+			[/\]/,    '@brackets.regexp.escape.control', '@pop'],
+		],
+
+		regexpcomment: [
+			[ /[^)]+/, 'comment' ],
+			[ /\)/, { token: 'regexp.escape.control', bracket: '@close', next: '@pop' } ]
+		],
+
+
+		// % quoted strings
+		// A bit repetitive since we need to often special case the kind of ending delimiter
+		pstring: [
+			[/%([qws])\(/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.(.)' } ],
+			[/%([qws])\[/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.[.]' } ],
+			[/%([qws])\{/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.{.}' } ],
+			[/%([qws])</,   { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.<.>' } ],
+			[/%([qws])(@delim)/, { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.$2.$2' } ],
+
+			[/%r\(/,  { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.(.)' } ],
+			[/%r\[/,  { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.[.]' } ],
+			[/%r\{/,  { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.{.}' } ],
+			[/%r</,   { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.<.>' } ],
+			[/%r(@delim)/, { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.$1.$1' } ],
+
+			[/%(x|W|Q?)\(/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.(.)' } ],
+			[/%(x|W|Q?)\[/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.[.]' } ],
+			[/%(x|W|Q?)\{/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.{.}' } ],
+			[/%(x|W|Q?)</,   { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.<.>' } ],
+			[/%(x|W|Q?)(@delim)/, { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.$2.$2' } ],
+
+			[/%([rqwsxW]|Q?)./, { token: 'invalid', next: '@pop' } ], // recover
+			[/./, { token: 'invalid', next: '@pop' } ], // recover
+		],
+
+		// non-expanded quoted string.
+		// qstring.<kind>.<open>.<close>
+		//  kind = q|w|s  (single quote, array, symbol)
+		//  open = open delimiter
+		//  close = close delimiter
+		qstring: [
+			[/\\$/, 'string.$S2.escape' ],
+			[/\\./, 'string.$S2.escape' ],
+			[/./,   { cases: { '$#==$S4' : { token: 'string.$S2.delim', bracket: '@close', next: '@pop' },
+								'$#==$S3' : { token: 'string.$S2.delim', bracket: '@open', next: '@push' }, // nested delimiters are allowed..
+								'@default': 'string.$S2' }}],
+		],
+
+		// expanded quoted string.
+		// qqstring.<kind>.<open>.<close>
+		//  kind = Q|W|x  (double quote, array, command)
+		//  open = open delimiter
+		//  close = close delimiter
+		qqstring: [
+			[/#/, 'string.$S2.escape', '@interpolated' ],
+			{ include: '@qstring' }
+		],
+
+
+		// whitespace & comments
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/^\s*=begin\b/,       'comment', '@comment' ],
+			[/#.*$/,    'comment'],
+		],
+
+		comment: [
+			[/[^=]+/, 'comment' ],
+			[/^\s*=begin\b/, 'comment.invalid' ],    // nested comment
+			[/^\s*=end\b.*/, 'comment', '@pop'  ],
+			[/[=]/, 'comment' ]
+		],
+	}
+};

src/sql.ts

@@ -0,0 +1,1132 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '--',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['[',']'],['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets:[
+	// 	{ openTrigger: 'n', open: /begin$/i, closeComplete: 'end', matchCase: true },
+	// 	{ openTrigger: 'e', open: /case$/i, closeComplete: 'end', matchCase: true },
+	// 	{ openTrigger: 'n', open: /when$/i, closeComplete: 'then', matchCase: true }
+	// ],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.sql',
+	ignoreCase: true,
+
+	brackets: [
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	keywords: [
+		'ABORT_AFTER_WAIT',
+		'ABSENT',
+		'ABSOLUTE',
+		'ACCENT_SENSITIVITY',
+		'ACTION',
+		'ACTIVATION',
+		'ACTIVE',
+		'ADD',
+		'ADDRESS',
+		'ADMIN',
+		'AES',
+		'AES_128',
+		'AES_192',
+		'AES_256',
+		'AFFINITY',
+		'AFTER',
+		'AGGREGATE',
+		'ALGORITHM',
+		'ALL_CONSTRAINTS',
+		'ALL_ERRORMSGS',
+		'ALL_INDEXES',
+		'ALL_LEVELS',
+		'ALL_SPARSE_COLUMNS',
+		'ALLOW_CONNECTIONS',
+		'ALLOW_MULTIPLE_EVENT_LOSS',
+		'ALLOW_PAGE_LOCKS',
+		'ALLOW_ROW_LOCKS',
+		'ALLOW_SINGLE_EVENT_LOSS',
+		'ALLOW_SNAPSHOT_ISOLATION',
+		'ALLOWED',
+		'ALTER',
+		'ANONYMOUS',
+		'ANSI_DEFAULTS',
+		'ANSI_NULL_DEFAULT',
+		'ANSI_NULL_DFLT_OFF',
+		'ANSI_NULL_DFLT_ON',
+		'ANSI_NULLS',
+		'ANSI_PADDING',
+		'ANSI_WARNINGS',
+		'APPEND',
+		'APPLICATION',
+		'APPLICATION_LOG',
+		'ARITHABORT',
+		'ARITHIGNORE',
+		'AS',
+		'ASC',
+		'ASSEMBLY',
+		'ASYMMETRIC',
+		'ASYNCHRONOUS_COMMIT',
+		'AT',
+		'ATOMIC',
+		'ATTACH',
+		'ATTACH_REBUILD_LOG',
+		'AUDIT',
+		'AUDIT_GUID',
+		'AUTHENTICATION',
+		'AUTHORIZATION',
+		'AUTO',
+		'AUTO_CLEANUP',
+		'AUTO_CLOSE',
+		'AUTO_CREATE_STATISTICS',
+		'AUTO_SHRINK',
+		'AUTO_UPDATE_STATISTICS',
+		'AUTO_UPDATE_STATISTICS_ASYNC',
+		'AUTOMATED_BACKUP_PREFERENCE',
+		'AUTOMATIC',
+		'AVAILABILITY',
+		'AVAILABILITY_MODE',
+		'BACKUP',
+		'BACKUP_PRIORITY',
+		'BASE64',
+		'BATCHSIZE',
+		'BEGIN',
+		'BEGIN_DIALOG',
+		'BIGINT',
+		'BINARY',
+		'BINDING',
+		'BIT',
+		'BLOCKERS',
+		'BLOCKSIZE',
+		'BOUNDING_BOX',
+		'BREAK',
+		'BROKER',
+		'BROKER_INSTANCE',
+		'BROWSE',
+		'BUCKET_COUNT',
+		'BUFFER',
+		'BUFFERCOUNT',
+		'BULK',
+		'BULK_LOGGED',
+		'BY',
+		'CACHE',
+		'CALL',
+		'CALLED',
+		'CALLER',
+		'CAP_CPU_PERCENT',
+		'CASCADE',
+		'CASE',
+		'CATALOG',
+		'CATCH',
+		'CELLS_PER_OBJECT',
+		'CERTIFICATE',
+		'CHANGE_RETENTION',
+		'CHANGE_TRACKING',
+		'CHANGES',
+		'CHAR',
+		'CHARACTER',
+		'CHECK',
+		'CHECK_CONSTRAINTS',
+		'CHECK_EXPIRATION',
+		'CHECK_POLICY',
+		'CHECKALLOC',
+		'CHECKCATALOG',
+		'CHECKCONSTRAINTS',
+		'CHECKDB',
+		'CHECKFILEGROUP',
+		'CHECKIDENT',
+		'CHECKPOINT',
+		'CHECKTABLE',
+		'CLASSIFIER_FUNCTION',
+		'CLEANTABLE',
+		'CLEANUP',
+		'CLEAR',
+		'CLOSE',
+		'CLUSTER',
+		'CLUSTERED',
+		'CODEPAGE',
+		'COLLATE',
+		'COLLECTION',
+		'COLUMN',
+		'COLUMN_SET',
+		'COLUMNS',
+		'COLUMNSTORE',
+		'COLUMNSTORE_ARCHIVE',
+		'COMMIT',
+		'COMMITTED',
+		'COMPATIBILITY_LEVEL',
+		'COMPRESSION',
+		'COMPUTE',
+		'CONCAT',
+		'CONCAT_NULL_YIELDS_NULL',
+		'CONFIGURATION',
+		'CONNECT',
+		'CONSTRAINT',
+		'CONTAINMENT',
+		'CONTENT',
+		'CONTEXT',
+		'CONTINUE',
+		'CONTINUE_AFTER_ERROR',
+		'CONTRACT',
+		'CONTRACT_NAME',
+		'CONTROL',
+		'CONVERSATION',
+		'COOKIE',
+		'COPY_ONLY',
+		'COUNTER',
+		'CPU',
+		'CREATE',
+		'CREATE_NEW',
+		'CREATION_DISPOSITION',
+		'CREDENTIAL',
+		'CRYPTOGRAPHIC',
+		'CUBE',
+		'CURRENT',
+		'CURRENT_DATE',
+		'CURSOR',
+		'CURSOR_CLOSE_ON_COMMIT',
+		'CURSOR_DEFAULT',
+		'CYCLE',
+		'DATA',
+		'DATA_COMPRESSION',
+		'DATA_PURITY',
+		'DATABASE',
+		'DATABASE_DEFAULT',
+		'DATABASE_MIRRORING',
+		'DATABASE_SNAPSHOT',
+		'DATAFILETYPE',
+		'DATE',
+		'DATE_CORRELATION_OPTIMIZATION',
+		'DATEFIRST',
+		'DATEFORMAT',
+		'DATETIME',
+		'DATETIME2',
+		'DATETIMEOFFSET',
+		'DAY',
+		'DAYOFYEAR',
+		'DAYS',
+		'DB_CHAINING',
+		'DBCC',
+		'DBREINDEX',
+		'DDL_DATABASE_LEVEL_EVENTS',
+		'DEADLOCK_PRIORITY',
+		'DEALLOCATE',
+		'DEC',
+		'DECIMAL',
+		'DECLARE',
+		'DECRYPTION',
+		'DEFAULT',
+		'DEFAULT_DATABASE',
+		'DEFAULT_FULLTEXT_LANGUAGE',
+		'DEFAULT_LANGUAGE',
+		'DEFAULT_SCHEMA',
+		'DEFINITION',
+		'DELAY',
+		'DELAYED_DURABILITY',
+		'DELETE',
+		'DELETED',
+		'DENSITY_VECTOR',
+		'DENY',
+		'DEPENDENTS',
+		'DES',
+		'DESC',
+		'DESCRIPTION',
+		'DESX',
+		'DHCP',
+		'DIAGNOSTICS',
+		'DIALOG',
+		'DIFFERENTIAL',
+		'DIRECTORY_NAME',
+		'DISABLE',
+		'DISABLE_BROKER',
+		'DISABLED',
+		'DISK',
+		'DISTINCT',
+		'DISTRIBUTED',
+		'DOCUMENT',
+		'DOUBLE',
+		'DROP',
+		'DROP_EXISTING',
+		'DROPCLEANBUFFERS',
+		'DUMP',
+		'DURABILITY',
+		'DYNAMIC',
+		'EDITION',
+		'ELEMENTS',
+		'ELSE',
+		'EMERGENCY',
+		'EMPTY',
+		'EMPTYFILE',
+		'ENABLE',
+		'ENABLE_BROKER',
+		'ENABLED',
+		'ENCRYPTION',
+		'END',
+		'ENDPOINT',
+		'ENDPOINT_URL',
+		'ERRLVL',
+		'ERROR',
+		'ERROR_BROKER_CONVERSATIONS',
+		'ERRORFILE',
+		'ESCAPE',
+		'ESTIMATEONLY',
+		'EVENT',
+		'EVENT_RETENTION_MODE',
+		'EXEC',
+		'EXECUTABLE',
+		'EXECUTE',
+		'EXIT',
+		'EXPAND',
+		'EXPIREDATE',
+		'EXPIRY_DATE',
+		'EXPLICIT',
+		'EXTENDED_LOGICAL_CHECKS',
+		'EXTENSION',
+		'EXTERNAL',
+		'EXTERNAL_ACCESS',
+		'FAIL_OPERATION',
+		'FAILOVER',
+		'FAILOVER_MODE',
+		'FAILURE_CONDITION_LEVEL',
+		'FALSE',
+		'FAN_IN',
+		'FAST',
+		'FAST_FORWARD',
+		'FETCH',
+		'FIELDTERMINATOR',
+		'FILE',
+		'FILEGROUP',
+		'FILEGROWTH',
+		'FILELISTONLY',
+		'FILENAME',
+		'FILEPATH',
+		'FILESTREAM',
+		'FILESTREAM_ON',
+		'FILETABLE_COLLATE_FILENAME',
+		'FILETABLE_DIRECTORY',
+		'FILETABLE_FULLPATH_UNIQUE_CONSTRAINT_NAME',
+		'FILETABLE_NAMESPACE',
+		'FILETABLE_PRIMARY_KEY_CONSTRAINT_NAME',
+		'FILETABLE_STREAMID_UNIQUE_CONSTRAINT_NAME',
+		'FILLFACTOR',
+		'FILTERING',
+		'FIRE_TRIGGERS',
+		'FIRST',
+		'FIRSTROW',
+		'FLOAT',
+		'FMTONLY',
+		'FOLLOWING',
+		'FOR',
+		'FORCE',
+		'FORCE_FAILOVER_ALLOW_DATA_LOSS',
+		'FORCE_SERVICE_ALLOW_DATA_LOSS',
+		'FORCED',
+		'FORCEPLAN',
+		'FORCESCAN',
+		'FORCESEEK',
+		'FOREIGN',
+		'FORMATFILE',
+		'FORMSOF',
+		'FORWARD_ONLY',
+		'FREE',
+		'FREEPROCCACHE',
+		'FREESESSIONCACHE',
+		'FREESYSTEMCACHE',
+		'FROM',
+		'FULL',
+		'FULLSCAN',
+		'FULLTEXT',
+		'FUNCTION',
+		'GB',
+		'GEOGRAPHY_AUTO_GRID',
+		'GEOGRAPHY_GRID',
+		'GEOMETRY_AUTO_GRID',
+		'GEOMETRY_GRID',
+		'GET',
+		'GLOBAL',
+		'GO',
+		'GOTO',
+		'GOVERNOR',
+		'GRANT',
+		'GRIDS',
+		'GROUP',
+		'GROUP_MAX_REQUESTS',
+		'HADR',
+		'HASH',
+		'HASHED',
+		'HAVING',
+		'HEADERONLY',
+		'HEALTH_CHECK_TIMEOUT',
+		'HELP',
+		'HIERARCHYID',
+		'HIGH',
+		'HINT',
+		'HISTOGRAM',
+		'HOLDLOCK',
+		'HONOR_BROKER_PRIORITY',
+		'HOUR',
+		'HOURS',
+		'IDENTITY',
+		'IDENTITY_INSERT',
+		'IDENTITY_VALUE',
+		'IDENTITYCOL',
+		'IF',
+		'IGNORE_CONSTRAINTS',
+		'IGNORE_DUP_KEY',
+		'IGNORE_NONCLUSTERED_COLUMNSTORE_INDEX',
+		'IGNORE_TRIGGERS',
+		'IMAGE',
+		'IMMEDIATE',
+		'IMPERSONATE',
+		'IMPLICIT_TRANSACTIONS',
+		'IMPORTANCE',
+		'INCLUDE',
+		'INCREMENT',
+		'INCREMENTAL',
+		'INDEX',
+		'INDEXDEFRAG',
+		'INFINITE',
+		'INFLECTIONAL',
+		'INIT',
+		'INITIATOR',
+		'INPUT',
+		'INPUTBUFFER',
+		'INSENSITIVE',
+		'INSERT',
+		'INSERTED',
+		'INSTEAD',
+		'INT',
+		'INTEGER',
+		'INTO',
+		'IO',
+		'IP',
+		'ISABOUT',
+		'ISOLATION',
+		'JOB',
+		'KB',
+		'KEEP',
+		'KEEP_CDC',
+		'KEEP_NULLS',
+		'KEEP_REPLICATION',
+		'KEEPDEFAULTS',
+		'KEEPFIXED',
+		'KEEPIDENTITY',
+		'KEEPNULLS',
+		'KERBEROS',
+		'KEY',
+		'KEY_SOURCE',
+		'KEYS',
+		'KEYSET',
+		'KILL',
+		'KILOBYTES_PER_BATCH',
+		'LABELONLY',
+		'LANGUAGE',
+		'LAST',
+		'LASTROW',
+		'LEVEL',
+		'LEVEL_1',
+		'LEVEL_2',
+		'LEVEL_3',
+		'LEVEL_4',
+		'LIFETIME',
+		'LINENO',
+		'LIST',
+		'LISTENER',
+		'LISTENER_IP',
+		'LISTENER_PORT',
+		'LOAD',
+		'LOADHISTORY',
+		'LOB_COMPACTION',
+		'LOCAL',
+		'LOCAL_SERVICE_NAME',
+		'LOCK_ESCALATION',
+		'LOCK_TIMEOUT',
+		'LOGIN',
+		'LOGSPACE',
+		'LOOP',
+		'LOW',
+		'MANUAL',
+		'MARK',
+		'MARK_IN_USE_FOR_REMOVAL',
+		'MASTER',
+		'MAX_CPU_PERCENT',
+		'MAX_DISPATCH_LATENCY',
+		'MAX_DOP',
+		'MAX_DURATION',
+		'MAX_EVENT_SIZE',
+		'MAX_FILES',
+		'MAX_IOPS_PER_VOLUME',
+		'MAX_MEMORY',
+		'MAX_MEMORY_PERCENT',
+		'MAX_QUEUE_READERS',
+		'MAX_ROLLOVER_FILES',
+		'MAX_SIZE',
+		'MAXDOP',
+		'MAXERRORS',
+		'MAXLENGTH',
+		'MAXRECURSION',
+		'MAXSIZE',
+		'MAXTRANSFERSIZE',
+		'MAXVALUE',
+		'MB',
+		'MEDIADESCRIPTION',
+		'MEDIANAME',
+		'MEDIAPASSWORD',
+		'MEDIUM',
+		'MEMBER',
+		'MEMORY_OPTIMIZED',
+		'MEMORY_OPTIMIZED_DATA',
+		'MEMORY_OPTIMIZED_ELEVATE_TO_SNAPSHOT',
+		'MEMORY_PARTITION_MODE',
+		'MERGE',
+		'MESSAGE',
+		'MESSAGE_FORWARD_SIZE',
+		'MESSAGE_FORWARDING',
+		'MICROSECOND',
+		'MILLISECOND',
+		'MIN_CPU_PERCENT',
+		'MIN_IOPS_PER_VOLUME',
+		'MIN_MEMORY_PERCENT',
+		'MINUTE',
+		'MINUTES',
+		'MINVALUE',
+		'MIRROR',
+		'MIRROR_ADDRESS',
+		'MODIFY',
+		'MONEY',
+		'MONTH',
+		'MOVE',
+		'MULTI_USER',
+		'MUST_CHANGE',
+		'NAME',
+		'NANOSECOND',
+		'NATIONAL',
+		'NATIVE_COMPILATION',
+		'NCHAR',
+		'NEGOTIATE',
+		'NESTED_TRIGGERS',
+		'NEW_ACCOUNT',
+		'NEW_BROKER',
+		'NEW_PASSWORD',
+		'NEWNAME',
+		'NEXT',
+		'NO',
+		'NO_BROWSETABLE',
+		'NO_CHECKSUM',
+		'NO_COMPRESSION',
+		'NO_EVENT_LOSS',
+		'NO_INFOMSGS',
+		'NO_TRUNCATE',
+		'NO_WAIT',
+		'NOCHECK',
+		'NOCOUNT',
+		'NOEXEC',
+		'NOEXPAND',
+		'NOFORMAT',
+		'NOINDEX',
+		'NOINIT',
+		'NOLOCK',
+		'NON',
+		'NON_TRANSACTED_ACCESS',
+		'NONCLUSTERED',
+		'NONE',
+		'NORECOMPUTE',
+		'NORECOVERY',
+		'NORESEED',
+		'NORESET',
+		'NOREWIND',
+		'NORMAL',
+		'NOSKIP',
+		'NOTIFICATION',
+		'NOTRUNCATE',
+		'NOUNLOAD',
+		'NOWAIT',
+		'NTEXT',
+		'NTLM',
+		'NUMANODE',
+		'NUMERIC',
+		'NUMERIC_ROUNDABORT',
+		'NVARCHAR',
+		'OBJECT',
+		'OF',
+		'OFF',
+		'OFFLINE',
+		'OFFSET',
+		'OFFSETS',
+		'OLD_ACCOUNT',
+		'OLD_PASSWORD',
+		'ON',
+		'ON_FAILURE',
+		'ONLINE',
+		'ONLY',
+		'OPEN',
+		'OPEN_EXISTING',
+		'OPENTRAN',
+		'OPTIMISTIC',
+		'OPTIMIZE',
+		'OPTION',
+		'ORDER',
+		'OUT',
+		'OUTPUT',
+		'OUTPUTBUFFER',
+		'OVER',
+		'OVERRIDE',
+		'OWNER',
+		'OWNERSHIP',
+		'PAD_INDEX',
+		'PAGE',
+		'PAGE_VERIFY',
+		'PAGECOUNT',
+		'PAGLOCK',
+		'PARAMETERIZATION',
+		'PARSEONLY',
+		'PARTIAL',
+		'PARTITION',
+		'PARTITIONS',
+		'PARTNER',
+		'PASSWORD',
+		'PATH',
+		'PER_CPU',
+		'PER_NODE',
+		'PERCENT',
+		'PERMISSION_SET',
+		'PERSISTED',
+		'PHYSICAL_ONLY',
+		'PLAN',
+		'POISON_MESSAGE_HANDLING',
+		'POOL',
+		'POPULATION',
+		'PORT',
+		'PRECEDING',
+		'PRECISION',
+		'PRIMARY',
+		'PRIMARY_ROLE',
+		'PRINT',
+		'PRIOR',
+		'PRIORITY',
+		'PRIORITY_LEVEL',
+		'PRIVATE',
+		'PRIVILEGES',
+		'PROC',
+		'PROCCACHE',
+		'PROCEDURE',
+		'PROCEDURE_NAME',
+		'PROCESS',
+		'PROFILE',
+		'PROPERTY',
+		'PROPERTY_DESCRIPTION',
+		'PROPERTY_INT_ID',
+		'PROPERTY_SET_GUID',
+		'PROVIDER',
+		'PROVIDER_KEY_NAME',
+		'PUBLIC',
+		'PUT',
+		'QUARTER',
+		'QUERY',
+		'QUERY_GOVERNOR_COST_LIMIT',
+		'QUEUE',
+		'QUEUE_DELAY',
+		'QUOTED_IDENTIFIER',
+		'RAISERROR',
+		'RANGE',
+		'RAW',
+		'RC2',
+		'RC4',
+		'RC4_128',
+		'READ',
+		'READ_COMMITTED_SNAPSHOT',
+		'READ_ONLY',
+		'READ_ONLY_ROUTING_LIST',
+		'READ_ONLY_ROUTING_URL',
+		'READ_WRITE',
+		'READ_WRITE_FILEGROUPS',
+		'READCOMMITTED',
+		'READCOMMITTEDLOCK',
+		'READONLY',
+		'READPAST',
+		'READTEXT',
+		'READUNCOMMITTED',
+		'READWRITE',
+		'REAL',
+		'REBUILD',
+		'RECEIVE',
+		'RECOMPILE',
+		'RECONFIGURE',
+		'RECOVERY',
+		'RECURSIVE',
+		'RECURSIVE_TRIGGERS',
+		'REFERENCES',
+		'REGENERATE',
+		'RELATED_CONVERSATION',
+		'RELATED_CONVERSATION_GROUP',
+		'RELATIVE',
+		'REMOTE',
+		'REMOTE_PROC_TRANSACTIONS',
+		'REMOTE_SERVICE_NAME',
+		'REMOVE',
+		'REORGANIZE',
+		'REPAIR_ALLOW_DATA_LOSS',
+		'REPAIR_FAST',
+		'REPAIR_REBUILD',
+		'REPEATABLE',
+		'REPEATABLEREAD',
+		'REPLICA',
+		'REPLICATION',
+		'REQUEST_MAX_CPU_TIME_SEC',
+		'REQUEST_MAX_MEMORY_GRANT_PERCENT',
+		'REQUEST_MEMORY_GRANT_TIMEOUT_SEC',
+		'REQUIRED',
+		'RESAMPLE',
+		'RESEED',
+		'RESERVE_DISK_SPACE',
+		'RESET',
+		'RESOURCE',
+		'RESTART',
+		'RESTORE',
+		'RESTRICT',
+		'RESTRICTED_USER',
+		'RESULT',
+		'RESUME',
+		'RETAINDAYS',
+		'RETENTION',
+		'RETURN',
+		'RETURNS',
+		'REVERT',
+		'REVOKE',
+		'REWIND',
+		'REWINDONLY',
+		'ROBUST',
+		'ROLE',
+		'ROLLBACK',
+		'ROLLUP',
+		'ROOT',
+		'ROUTE',
+		'ROW',
+		'ROWCOUNT',
+		'ROWGUIDCOL',
+		'ROWLOCK',
+		'ROWS',
+		'ROWS_PER_BATCH',
+		'ROWTERMINATOR',
+		'ROWVERSION',
+		'RSA_1024',
+		'RSA_2048',
+		'RSA_512',
+		'RULE',
+		'SAFE',
+		'SAFETY',
+		'SAMPLE',
+		'SAVE',
+		'SCHEDULER',
+		'SCHEMA',
+		'SCHEMA_AND_DATA',
+		'SCHEMA_ONLY',
+		'SCHEMABINDING',
+		'SCHEME',
+		'SCROLL',
+		'SCROLL_LOCKS',
+		'SEARCH',
+		'SECOND',
+		'SECONDARY',
+		'SECONDARY_ONLY',
+		'SECONDARY_ROLE',
+		'SECONDS',
+		'SECRET',
+		'SECURITY_LOG',
+		'SECURITYAUDIT',
+		'SELECT',
+		'SELECTIVE',
+		'SELF',
+		'SEND',
+		'SENT',
+		'SEQUENCE',
+		'SERIALIZABLE',
+		'SERVER',
+		'SERVICE',
+		'SERVICE_BROKER',
+		'SERVICE_NAME',
+		'SESSION',
+		'SESSION_TIMEOUT',
+		'SET',
+		'SETS',
+		'SETUSER',
+		'SHOW_STATISTICS',
+		'SHOWCONTIG',
+		'SHOWPLAN',
+		'SHOWPLAN_ALL',
+		'SHOWPLAN_TEXT',
+		'SHOWPLAN_XML',
+		'SHRINKDATABASE',
+		'SHRINKFILE',
+		'SHUTDOWN',
+		'SID',
+		'SIGNATURE',
+		'SIMPLE',
+		'SINGLE_BLOB',
+		'SINGLE_CLOB',
+		'SINGLE_NCLOB',
+		'SINGLE_USER',
+		'SINGLETON',
+		'SIZE',
+		'SKIP',
+		'SMALLDATETIME',
+		'SMALLINT',
+		'SMALLMONEY',
+		'SNAPSHOT',
+		'SORT_IN_TEMPDB',
+		'SOURCE',
+		'SPARSE',
+		'SPATIAL',
+		'SPATIAL_WINDOW_MAX_CELLS',
+		'SPECIFICATION',
+		'SPLIT',
+		'SQL',
+		'SQL_VARIANT',
+		'SQLPERF',
+		'STANDBY',
+		'START',
+		'START_DATE',
+		'STARTED',
+		'STARTUP_STATE',
+		'STAT_HEADER',
+		'STATE',
+		'STATEMENT',
+		'STATIC',
+		'STATISTICAL_SEMANTICS',
+		'STATISTICS',
+		'STATISTICS_INCREMENTAL',
+		'STATISTICS_NORECOMPUTE',
+		'STATS',
+		'STATS_STREAM',
+		'STATUS',
+		'STATUSONLY',
+		'STOP',
+		'STOP_ON_ERROR',
+		'STOPAT',
+		'STOPATMARK',
+		'STOPBEFOREMARK',
+		'STOPLIST',
+		'STOPPED',
+		'SUBJECT',
+		'SUBSCRIPTION',
+		'SUPPORTED',
+		'SUSPEND',
+		'SWITCH',
+		'SYMMETRIC',
+		'SYNCHRONOUS_COMMIT',
+		'SYNONYM',
+		'SYSNAME',
+		'SYSTEM',
+		'TABLE',
+		'TABLERESULTS',
+		'TABLESAMPLE',
+		'TABLOCK',
+		'TABLOCKX',
+		'TAKE',
+		'TAPE',
+		'TARGET',
+		'TARGET_RECOVERY_TIME',
+		'TB',
+		'TCP',
+		'TEXT',
+		'TEXTIMAGE_ON',
+		'TEXTSIZE',
+		'THEN',
+		'THESAURUS',
+		'THROW',
+		'TIES',
+		'TIME',
+		'TIMEOUT',
+		'TIMER',
+		'TIMESTAMP',
+		'TINYINT',
+		'TO',
+		'TOP',
+		'TORN_PAGE_DETECTION',
+		'TRACEOFF',
+		'TRACEON',
+		'TRACESTATUS',
+		'TRACK_CAUSALITY',
+		'TRACK_COLUMNS_UPDATED',
+		'TRAN',
+		'TRANSACTION',
+		'TRANSFER',
+		'TRANSFORM_NOISE_WORDS',
+		'TRIGGER',
+		'TRIPLE_DES',
+		'TRIPLE_DES_3KEY',
+		'TRUE',
+		'TRUNCATE',
+		'TRUNCATEONLY',
+		'TRUSTWORTHY',
+		'TRY',
+		'TSQL',
+		'TWO_DIGIT_YEAR_CUTOFF',
+		'TYPE',
+		'TYPE_WARNING',
+		'UNBOUNDED',
+		'UNCHECKED',
+		'UNCOMMITTED',
+		'UNDEFINED',
+		'UNIQUE',
+		'UNIQUEIDENTIFIER',
+		'UNKNOWN',
+		'UNLIMITED',
+		'UNLOAD',
+		'UNSAFE',
+		'UPDATE',
+		'UPDATETEXT',
+		'UPDATEUSAGE',
+		'UPDLOCK',
+		'URL',
+		'USE',
+		'USED',
+		'USER',
+		'USEROPTIONS',
+		'USING',
+		'VALID_XML',
+		'VALIDATION',
+		'VALUE',
+		'VALUES',
+		'VARBINARY',
+		'VARCHAR',
+		'VARYING',
+		'VERIFYONLY',
+		'VERSION',
+		'VIEW',
+		'VIEW_METADATA',
+		'VIEWS',
+		'VISIBILITY',
+		'WAIT_AT_LOW_PRIORITY',
+		'WAITFOR',
+		'WEEK',
+		'WEIGHT',
+		'WELL_FORMED_XML',
+		'WHEN',
+		'WHERE',
+		'WHILE',
+		'WINDOWS',
+		'WITH',
+		'WITHIN',
+		'WITHOUT',
+		'WITNESS',
+		'WORK',
+		'WORKLOAD',
+		'WRITETEXT',
+		'XACT_ABORT',
+		'XLOCK',
+		'XMAX',
+		'XMIN',
+		'XML',
+		'XMLDATA',
+		'XMLNAMESPACES',
+		'XMLSCHEMA',
+		'XQUERY',
+		'XSINIL',
+		'YEAR',
+		'YMAX',
+		'YMIN'
+	],
+	operators: [
+		// Logical
+		'ALL','AND','ANY','BETWEEN','EXISTS','IN','LIKE','NOT','OR','SOME',
+		// Set
+		'EXCEPT','INTERSECT','UNION',
+		// Join
+		'APPLY','CROSS','FULL','INNER','JOIN','LEFT','OUTER','RIGHT',
+		// Predicates
+		'CONTAINS','FREETEXT','IS','NULL',
+		// Pivoting
+		'PIVOT','UNPIVOT',
+		// Merging
+		'MATCHED'
+	],
+	builtinFunctions: [
+		// Aggregate
+		'AVG','CHECKSUM_AGG','COUNT','COUNT_BIG','GROUPING','GROUPING_ID','MAX','MIN','SUM','STDEV','STDEVP','VAR','VARP',
+		// Analytic
+		'CUME_DIST','FIRST_VALUE','LAG','LAST_VALUE','LEAD','PERCENTILE_CONT','PERCENTILE_DISC','PERCENT_RANK',
+		// Collation
+		'COLLATE','COLLATIONPROPERTY','TERTIARY_WEIGHTS',
+		// Azure
+		'FEDERATION_FILTERING_VALUE',
+		// Conversion
+		'CAST','CONVERT','PARSE','TRY_CAST','TRY_CONVERT','TRY_PARSE',
+		// Cryptographic
+		'ASYMKEY_ID','ASYMKEYPROPERTY','CERTPROPERTY','CERT_ID','CRYPT_GEN_RANDOM',
+		'DECRYPTBYASYMKEY','DECRYPTBYCERT','DECRYPTBYKEY','DECRYPTBYKEYAUTOASYMKEY','DECRYPTBYKEYAUTOCERT','DECRYPTBYPASSPHRASE',
+		'ENCRYPTBYASYMKEY','ENCRYPTBYCERT','ENCRYPTBYKEY','ENCRYPTBYPASSPHRASE','HASHBYTES','IS_OBJECTSIGNED',
+		'KEY_GUID','KEY_ID','KEY_NAME','SIGNBYASYMKEY','SIGNBYCERT','SYMKEYPROPERTY','VERIFYSIGNEDBYCERT','VERIFYSIGNEDBYASYMKEY',
+		// Cursor
+		'CURSOR_STATUS',
+		// Datatype
+		'DATALENGTH','IDENT_CURRENT','IDENT_INCR','IDENT_SEED','IDENTITY','SQL_VARIANT_PROPERTY',
+		// Datetime
+		'CURRENT_TIMESTAMP','DATEADD','DATEDIFF','DATEFROMPARTS','DATENAME','DATEPART','DATETIME2FROMPARTS','DATETIMEFROMPARTS',
+		'DATETIMEOFFSETFROMPARTS','DAY','EOMONTH','GETDATE','GETUTCDATE','ISDATE','MONTH','SMALLDATETIMEFROMPARTS','SWITCHOFFSET',
+		'SYSDATETIME','SYSDATETIMEOFFSET','SYSUTCDATETIME','TIMEFROMPARTS','TODATETIMEOFFSET','YEAR',
+		// Logical
+		'CHOOSE','COALESCE','IIF','NULLIF',
+		// Mathematical
+		'ABS','ACOS','ASIN','ATAN','ATN2','CEILING','COS','COT','DEGREES','EXP','FLOOR','LOG','LOG10',
+		'PI','POWER','RADIANS','RAND','ROUND','SIGN','SIN','SQRT','SQUARE','TAN',
+		// Metadata
+		'APP_NAME','APPLOCK_MODE','APPLOCK_TEST','ASSEMBLYPROPERTY','COL_LENGTH','COL_NAME','COLUMNPROPERTY',
+		'DATABASE_PRINCIPAL_ID','DATABASEPROPERTYEX','DB_ID','DB_NAME','FILE_ID','FILE_IDEX','FILE_NAME','FILEGROUP_ID',
+		'FILEGROUP_NAME','FILEGROUPPROPERTY','FILEPROPERTY','FULLTEXTCATALOGPROPERTY','FULLTEXTSERVICEPROPERTY',
+		'INDEX_COL','INDEXKEY_PROPERTY','INDEXPROPERTY','OBJECT_DEFINITION','OBJECT_ID',
+		'OBJECT_NAME','OBJECT_SCHEMA_NAME','OBJECTPROPERTY','OBJECTPROPERTYEX','ORIGINAL_DB_NAME','PARSENAME',
+		'SCHEMA_ID','SCHEMA_NAME','SCOPE_IDENTITY','SERVERPROPERTY','STATS_DATE','TYPE_ID','TYPE_NAME','TYPEPROPERTY',
+		// Ranking
+		'DENSE_RANK','NTILE','RANK','ROW_NUMBER',
+		// Replication
+		'PUBLISHINGSERVERNAME',
+		// Rowset
+		'OPENDATASOURCE','OPENQUERY','OPENROWSET','OPENXML',
+		// Security
+		'CERTENCODED','CERTPRIVATEKEY','CURRENT_USER','HAS_DBACCESS','HAS_PERMS_BY_NAME','IS_MEMBER','IS_ROLEMEMBER','IS_SRVROLEMEMBER',
+		'LOGINPROPERTY','ORIGINAL_LOGIN','PERMISSIONS','PWDENCRYPT','PWDCOMPARE','SESSION_USER','SESSIONPROPERTY','SUSER_ID','SUSER_NAME',
+		'SUSER_SID','SUSER_SNAME','SYSTEM_USER','USER','USER_ID','USER_NAME',
+		// String
+		'ASCII','CHAR','CHARINDEX','CONCAT','DIFFERENCE','FORMAT','LEFT','LEN','LOWER','LTRIM','NCHAR','PATINDEX',
+		'QUOTENAME','REPLACE','REPLICATE','REVERSE','RIGHT','RTRIM','SOUNDEX','SPACE','STR','STUFF','SUBSTRING','UNICODE','UPPER',
+		// System
+		'BINARY_CHECKSUM','CHECKSUM','CONNECTIONPROPERTY','CONTEXT_INFO','CURRENT_REQUEST_ID','ERROR_LINE','ERROR_NUMBER','ERROR_MESSAGE',
+		'ERROR_PROCEDURE','ERROR_SEVERITY','ERROR_STATE','FORMATMESSAGE','GETANSINULL','GET_FILESTREAM_TRANSACTION_CONTEXT','HOST_ID',
+		'HOST_NAME','ISNULL','ISNUMERIC','MIN_ACTIVE_ROWVERSION','NEWID','NEWSEQUENTIALID','ROWCOUNT_BIG','XACT_STATE',
+		// TextImage
+		'TEXTPTR','TEXTVALID',
+		// Trigger
+		'COLUMNS_UPDATED','EVENTDATA','TRIGGER_NESTLEVEL','UPDATE',
+		// ChangeTracking
+		'CHANGETABLE','CHANGE_TRACKING_CONTEXT','CHANGE_TRACKING_CURRENT_VERSION','CHANGE_TRACKING_IS_COLUMN_IN_MASK','CHANGE_TRACKING_MIN_VALID_VERSION',
+		// FullTextSearch
+		'CONTAINSTABLE','FREETEXTTABLE',
+		// SemanticTextSearch
+		'SEMANTICKEYPHRASETABLE','SEMANTICSIMILARITYDETAILSTABLE','SEMANTICSIMILARITYTABLE',
+		// FileStream
+		'FILETABLEROOTPATH','GETFILENAMESPACEPATH','GETPATHLOCATOR','PATHNAME',
+		// ServiceBroker
+		'GET_TRANSMISSION_STATUS'
+	],
+	builtinVariables: [
+		// Configuration
+		'@@DATEFIRST','@@DBTS','@@LANGID','@@LANGUAGE','@@LOCK_TIMEOUT','@@MAX_CONNECTIONS','@@MAX_PRECISION','@@NESTLEVEL',
+		'@@OPTIONS','@@REMSERVER','@@SERVERNAME','@@SERVICENAME','@@SPID','@@TEXTSIZE','@@VERSION',
+		// Cursor
+		'@@CURSOR_ROWS','@@FETCH_STATUS',
+		// Datetime
+		'@@DATEFIRST',
+		// Metadata
+		'@@PROCID',
+		// System
+		'@@ERROR','@@IDENTITY','@@ROWCOUNT','@@TRANCOUNT',
+		// Stats
+		'@@CONNECTIONS','@@CPU_BUSY','@@IDLE','@@IO_BUSY','@@PACKET_ERRORS','@@PACK_RECEIVED','@@PACK_SENT',
+		'@@TIMETICKS','@@TOTAL_ERRORS','@@TOTAL_READ','@@TOTAL_WRITE'
+	],
+	pseudoColumns: [
+		'$ACTION', '$IDENTITY', '$ROWGUID', '$PARTITION'
+	],
+	tokenizer: {
+		root: [
+			{ include: '@comments' },
+			{ include: '@whitespace' },
+			{ include: '@pseudoColumns' },
+			{ include: '@numbers' },
+			{ include: '@strings' },
+			{ include: '@complexIdentifiers' },
+			{ include: '@scopes' },
+			[/[;,.]/, 'delimiter'],
+			[/[()]/, '@brackets'],
+			[/[\w@#$]+/, { cases: {
+							'@keywords': 'keyword',
+							'@operators': 'operator',
+							'@builtinVariables': 'predefined',
+							'@builtinFunctions': 'predefined',
+							'@default': 'identifier' }
+						}],
+			[/[<>=!%&+\-*/|~^]/, 'operator'],
+		],
+		whitespace: [
+			[/\s+/, 'white']
+		],
+		comments: [
+			[/--+.*/, 'comment'],
+			[/\/\*/, { token: 'comment.quote', bracket: '@open', next: '@comment' }]
+		],
+		comment: [
+			[/[^*/]+/, 'comment'],
+			// Not supporting nested comments, as nested comments seem to not be standard?
+			// i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic
+			// [/\/\*/, { token: 'comment.quote', bracket: '@open', next: '@push' }],    // nested comment not allowed :-(
+			[/\*\//, { token: 'comment.quote', bracket: '@close', next: '@pop' }],
+			[/./, 'comment']
+		],
+		pseudoColumns: [
+			[/[$][A-Za-z_][\w@#$]*/, { cases: {
+				'@pseudoColumns': 'predefined',
+				'@default': 'identifier'
+			}}],
+		],
+		numbers: [
+			[/0[xX][0-9a-fA-F]*/, 'number'],
+			[/[$][+-]*\d*(\.\d*)?/, 'number'],
+			[/((\d+(\.\d*)?)|(\.\d+))([eE][\-+]?\d+)?/, 'number']
+		],
+		strings: [
+			[/N'/, { token: 'string.quote', bracket: '@open', next: '@string' }],
+			[/'/, { token: 'string.quote', bracket: '@open', next: '@string' }]
+		],
+		string: [
+			[/[^']+/, 'string'],
+			[/''/, 'string'],
+			[/'/, { token: 'string.quote', bracket: '@close', next: '@pop' }]
+		],
+		complexIdentifiers: [
+			[/\[/, { token: 'identifier.quote', bracket: '@open', next: '@bracketedIdentifier' }],
+			[/"/, { token: 'identifier.quote', bracket: '@open', next: '@quotedIdentifier' }]
+		],
+		bracketedIdentifier: [
+			[/[^\]]+/, 'identifier'],
+			[/]]/, 'identifier'],
+			[/]/, { token: 'identifier.quote', bracket: '@close', next: '@pop' }]
+		],
+		quotedIdentifier: [
+			[/[^"]+/, 'identifier'],
+			[/""/, 'identifier'],
+			[/"/, { token: 'identifier.quote', bracket: '@close', next: '@pop' }]
+		],
+		scopes: [
+			[/BEGIN\s+(DISTRIBUTED\s+)?TRAN(SACTION)?\b/i, 'keyword'],
+			[/BEGIN\s+TRY\b/i, { token: 'keyword.try', bracket: '@open' }],
+			[/END\s+TRY\b/i, { token: 'keyword.try', bracket: '@close' }],
+			[/BEGIN\s+CATCH\b/i, { token: 'keyword.catch', bracket: '@open' }],
+			[/END\s+CATCH\b/i, { token: 'keyword.catch', bracket: '@close' }],
+			[/(BEGIN|CASE)\b/i, { token: 'keyword.block', bracket: '@open' }],
+			[/END\b/i, { token: 'keyword.block', bracket: '@close' }],
+			[/WHEN\b/i, { token: 'keyword.choice', bracket: '@open' }],
+			[/THEN\b/i, { token: 'keyword.choice', bracket: '@close' }]
+		]
+	}
+};

src/swift.ts

@@ -0,0 +1,143 @@
+/*!---------------------------------------------------------------------------------------------
+ *  Copyright (C) David Owens II, owensd.io. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'],['[',']'],['(',')'],['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.swift',
+
+	// TODO(owensd): Support the full range of unicode valid identifiers.
+	identifier: /[a-zA-Z_][\w$]*/,
+	// TODO(owensd): Support the @availability macro properly.
+	attributes: [
+		'@autoclosure', '@noescape', '@noreturn', '@NSApplicationMain', '@NSCopying', '@NSManaged',
+		'@objc', '@UIApplicationMain', '@noreturn', '@availability', '@IBAction', '@IBDesignable', '@IBInspectable', '@IBOutlet'
+	],
+	accessmodifiers: [ 'public', 'private', 'internal' ],
+	keywords: [
+		'__COLUMN__', '__FILE__', '__FUNCTION__', '__LINE__', 'as', 'as!', 'as?', 'associativity', 'break', 'case', 'catch',
+		'class', 'continue', 'convenience', 'default', 'deinit', 'didSet', 'do', 'dynamic', 'dynamicType',
+		'else', 'enum', 'extension', 'fallthrough', 'final', 'for', 'func', 'get', 'guard', 'if', 'import', 'in', 'infix',
+		'init', 'inout', 'internal', 'is', 'lazy', 'left', 'let', 'mutating', 'nil', 'none', 'nonmutating', 'operator',
+		'optional', 'override', 'postfix', 'precedence', 'prefix', 'private', 'protocol', 'Protocol', 'public',
+		'repeat', 'required', 'return', 'right', 'self', 'Self', 'set', 'static', 'struct', 'subscript', 'super', 'switch',
+		'throw', 'throws', 'try', 'try!', 'Type', 'typealias', 'unowned', 'var', 'weak', 'where', 'while', 'willSet', 'FALSE', 'TRUE'
+	],
+
+	symbols: /[=(){}\[\].,:;@#\_&\-<>`?!+*\\\/]/,
+
+	// Moved . to operatorstart so it can be a delimiter
+	operatorstart: /[\/=\-+!*%<>&|^~?\u00A1-\u00A7\u00A9\u00AB\u00AC\u00AE\u00B0-\u00B1\u00B6\u00BB\u00BF\u00D7\u00F7\u2016-\u2017\u2020-\u2027\u2030-\u203E\u2041-\u2053\u2055-\u205E\u2190-\u23FF\u2500-\u2775\u2794-\u2BFF\u2E00-\u2E7F\u3001-\u3003\u3008-\u3030]/,
+	operatorend: /[\u0300-\u036F\u1DC0-\u1DFF\u20D0-\u20FF\uFE00-\uFE0F\uFE20-\uFE2F\uE0100-\uE01EF]/,
+	operators: /(@operatorstart)((@operatorstart)|(@operatorend))*/,
+
+	// TODO(owensd): These are borrowed from C#; need to validate correctness for Swift.
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	tokenizer: {
+		root: [
+			{ include: '@comment' },
+			{ include: '@attribute' },
+			{ include: '@literal' },
+			{ include: '@keyword' },
+			{ include: '@invokedmethod' },
+			{ include: '@symbol' },
+		],
+
+		symbol: [
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/[.]/, 'delimiter'],
+			[/@operators/, 'keyword.operator'],
+			[/@symbols/, 'keyword.operator']
+		],
+
+
+		comment: [
+			[ /\/\/\/.*$/, 'comment.doc' ],
+			[ /\/\*\*/, 'comment.doc', '@commentdocbody' ],
+			[ /\/\/.*$/, 'comment' ],
+			[ /\/\*/, 'comment', '@commentbody' ]
+		],
+		commentdocbody: [
+			[ /\/\*/, 'comment', '@commentbody' ],
+			[ /\*\//, 'comment.doc', '@pop' ],
+			[ /\:[a-zA-Z]+\:/, 'comment.doc.param' ],
+			[ /./, 'comment.doc' ]
+		],
+		commentbody: [
+			[ /\/\*/, 'comment', '@commentbody' ],
+			[ /\*\//, 'comment', '@pop' ],
+			[ /./, 'comment' ]
+		],
+
+		attribute: [
+			[ /\@@identifier/, { cases: { '@attributes': 'keyword.control', '@default': '' } } ]
+		],
+
+		literal: [
+			[ /"/, { token: 'string.quote', bracket: '@open', next: '@stringlit' } ],
+			[ /0[b]([01]_?)+/, 'number.binary' ],
+			[ /0[o]([0-7]_?)+/, 'number.octal' ],
+			[ /0[x]([0-9a-fA-F]_?)+([pP][\-+](\d_?)+)?/, 'number.hex' ],
+			[ /(\d_?)*\.(\d_?)+([eE][\-+]?(\d_?)+)?/, 'number.float'],
+			[ /(\d_?)+/, 'number' ]
+		],
+
+		stringlit: [
+			[ /\\\(/, { token: 'keyword.operator', bracket: '@open', next: '@interpolatedexpression' } ],
+			[ /@escapes/, 'string' ],
+			[ /\\./, 'string.escape.invalid' ],
+			[ /"/, { token: 'string.quote', bracket: '@close', next: '@pop' } ],
+			[ /./, 'string' ]
+		],
+
+		interpolatedexpression: [
+			[ /\(/, { token: 'keyword.operator', bracket: '@open', next: '@interpolatedexpression' } ],
+			[ /\)/, { token: 'keyword.operator', bracket: '@close', next: '@pop' } ],
+			{ include: '@literal' },
+			{ include: '@keyword' },
+			{ include: '@symbol' }
+		],
+
+		keyword: [
+			[ /`/, { token: 'keyword.operator', bracket: '@open', next: '@escapedkeyword' } ],
+			[ /@identifier/, { cases: { '@keywords': 'keyword', '[A-Z][\a-zA-Z0-9$]*': 'type.identifier', '@default': 'identifier' } }]
+		],
+
+		escapedkeyword: [
+			[ /`/, { token: 'keyword.operator', bracket: '@close', next: '@pop' } ],
+			[ /./, 'identifier' ]
+		],
+
+//		symbol: [
+//			[ /@symbols/, 'keyword.operator' ],
+//			[ /@operators/, 'keyword.operator' ]
+//		],
+
+		invokedmethod: [
+			[/([.])(@identifier)/, { cases: { '$2': ['delimeter', 'type.identifier'], '@default': '' } }],
+		]
+	}
+};

src/vb.ts

@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '\'',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [
+		['{','}'],['[',']'],['(',')'],['<','>'],
+		['addhandler','end addhandler'],
+		['class','end class'],
+		['enum','end enum'],
+		['event','end event'],
+		['function','end function'],
+		['get','end get'],
+		['if','end if'],
+		['interface','end interface'],
+		['module','end module'],
+		['namespace','end namespace'],
+		['operator','end operator'],
+		['property','end property'],
+		['raiseevent','end raiseevent'],
+		['removehandler','end removehandler'],
+		['select','end select'],
+		['set','end set'],
+		['structure','end structure'],
+		['sub','end sub'],
+		['synclock','end synclock'],
+		['try','end try'],
+		['while','end while'],
+		['with','end with'],
+		['using','end using'],
+		['do','loop'],
+		['for','next']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.vb',
+	ignoreCase: true,
+
+	brackets: [
+		{ token:'delimiter.bracket', open: '{', close: '}'},
+		{ token:'delimiter.array', open: '[', close: ']'},
+		{ token:'delimiter.parenthesis', open: '(', close: ')'},
+		{ token:'delimiter.angle', open: '<', close: '>'},
+
+		// Special bracket statement pairs
+		// according to https://msdn.microsoft.com/en-us/library/tsw2a11z.aspx
+		{ token: 'keyword.tag-addhandler', open: 'addhandler', close: 'end addhandler'},
+		{ token: 'keyword.tag-class', open: 'class', close: 'end class'},
+		{ token: 'keyword.tag-enum', open: 'enum', close: 'end enum'},
+		{ token: 'keyword.tag-event', open: 'event', close: 'end event'},
+		{ token: 'keyword.tag-function', open: 'function', close: 'end function'},
+		{ token: 'keyword.tag-get', open: 'get', close: 'end get'},
+		{ token: 'keyword.tag-if', open: 'if', close: 'end if'},
+		{ token: 'keyword.tag-interface', open: 'interface', close: 'end interface'},
+		{ token: 'keyword.tag-module', open: 'module', close: 'end module'},
+		{ token: 'keyword.tag-namespace', open: 'namespace', close: 'end namespace'},
+		{ token: 'keyword.tag-operator', open: 'operator', close: 'end operator'},
+		{ token: 'keyword.tag-property', open: 'property', close: 'end property'},
+		{ token: 'keyword.tag-raiseevent', open: 'raiseevent', close: 'end raiseevent'},
+		{ token: 'keyword.tag-removehandler', open: 'removehandler', close: 'end removehandler'},
+		{ token: 'keyword.tag-select', open: 'select', close: 'end select'},
+		{ token: 'keyword.tag-set', open: 'set', close: 'end set'},
+		{ token: 'keyword.tag-structure', open: 'structure', close: 'end structure'},
+		{ token: 'keyword.tag-sub', open: 'sub', close: 'end sub'},
+		{ token: 'keyword.tag-synclock', open: 'synclock', close: 'end synclock'},
+		{ token: 'keyword.tag-try', open: 'try', close: 'end try'},
+		{ token: 'keyword.tag-while', open: 'while', close: 'end while'},
+		{ token: 'keyword.tag-with', open: 'with', close: 'end with'},
+
+		// Other pairs
+		{ token: 'keyword.tag-using', open: 'using', close: 'end using' },
+		{ token: 'keyword.tag-do', open: 'do', close: 'loop' },
+		{ token: 'keyword.tag-for', open: 'for', close: 'next' }
+	],
+
+	keywords: [
+		'AddHandler', 'AddressOf', 'Alias', 'And', 'AndAlso', 'As', 'Async', 'Boolean', 'ByRef', 'Byte', 'ByVal', 'Call',
+		'Case', 'Catch', 'CBool', 'CByte', 'CChar', 'CDate', 'CDbl', 'CDec', 'Char', 'CInt', 'Class', 'CLng',
+		'CObj', 'Const', 'Continue', 'CSByte', 'CShort', 'CSng', 'CStr', 'CType', 'CUInt', 'CULng', 'CUShort',
+		'Date', 'Decimal', 'Declare', 'Default', 'Delegate', 'Dim', 'DirectCast', 'Do', 'Double', 'Each', 'Else',
+		'ElseIf', 'End', 'EndIf', 'Enum', 'Erase', 'Error', 'Event', 'Exit', 'False', 'Finally', 'For', 'Friend',
+		'Function', 'Get', 'GetType', 'GetXMLNamespace', 'Global', 'GoSub', 'GoTo', 'Handles', 'If', 'Implements',
+		'Imports', 'In', 'Inherits', 'Integer', 'Interface', 'Is', 'IsNot', 'Let', 'Lib', 'Like', 'Long', 'Loop',
+		'Me', 'Mod', 'Module', 'MustInherit', 'MustOverride', 'MyBase', 'MyClass', 'NameOf', 'Namespace', 'Narrowing', 'New',
+		'Next', 'Not', 'Nothing', 'NotInheritable', 'NotOverridable', 'Object', 'Of', 'On', 'Operator', 'Option',
+		'Optional', 'Or', 'OrElse', 'Out', 'Overloads', 'Overridable', 'Overrides', 'ParamArray', 'Partial',
+		'Private', 'Property', 'Protected', 'Public', 'RaiseEvent', 'ReadOnly', 'ReDim', 'RemoveHandler', 'Resume',
+		'Return', 'SByte', 'Select', 'Set', 'Shadows', 'Shared', 'Short', 'Single', 'Static', 'Step', 'Stop',
+		'String', 'Structure', 'Sub', 'SyncLock', 'Then', 'Throw', 'To', 'True', 'Try', 'TryCast', 'TypeOf',
+		'UInteger', 'ULong', 'UShort', 'Using', 'Variant', 'Wend', 'When', 'While', 'Widening', 'With', 'WithEvents',
+		'WriteOnly', 'Xor'
+	],
+
+	tagwords: [
+		'If', 'Sub', 'Select', 'Try', 'Class', 'Enum',
+		'Function', 'Get', 'Interface', 'Module', 'Namespace', 'Operator', 'Set', 'Structure', 'Using', 'While', 'With',
+		'Do', 'Loop', 'For', 'Next', 'Property', 'Continue', 'AddHandler', 'RemoveHandler', 'Event', 'RaiseEvent', 'SyncLock'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?;\.,:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /U?[DI%L&S@]?/,
+	floatsuffix: /[R#F!]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// special ending tag-words
+			[/next(?!\w)/, { token: 'keyword.tag-for', bracket: '@close'}],
+			[/loop(?!\w)/, { token: 'keyword.tag-do', bracket: '@close' }],
+
+			// usual ending tags
+			[/end\s+(?!for|do)([a-zA-Z_]\w*)/, { token: 'keyword.tag-$1', bracket: '@close' }],
+
+			// identifiers, tagwords, and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@tagwords': {token:'keyword.tag-$0', bracket: '@open'},
+										'@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// numbers
+			[/\d*\d+e([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+(e[\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/&H[0-9a-f]+(@integersuffix)/, 'number.hex'],
+			[/&0[0-7]+(@integersuffix)/, 'number.octal'],
+			[/\d+(@integersuffix)/, 'number'],
+
+			// date literal
+			[/#.*#/, 'number'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/(\'|REM(?!\w)).*$/,        'comment'],
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"C?/,        'string', '@pop' ]
+		],
+	},
+};

src/xml.ts

@@ -0,0 +1,104 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		blockComment: ['<!--', '-->'],
+	},
+	brackets: [['{','}'],['[',']'],['(',')'],['<','>']],
+	autoClosingPairs: [
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [{
+	// 	tokenType: 'tag.tag-$1.xml',
+	// 	openTrigger: '>',
+	// 	open: /<(\w[\w\d]*)([^\/>]*(?!\/)>)[^<>]*$/i,
+	// 	closeComplete: '</$1>',
+	// 	closeTrigger: '>',
+	// 	close: /<\/(\w[\w\d]*)\s*>$/i
+	// }],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.xml',
+
+	ignoreCase: true,
+
+	// Useful regular expressions
+	qualifiedName: /(?:[\w\.\-]+:)?[\w\.\-]+/,
+
+	tokenizer: {
+		root: [
+			[/[^<&]+/, ''],
+
+			{ include: '@whitespace' },
+
+			// Standard opening tag
+			[/(<)(@qualifiedName)/, [
+				{ token: 'delimiter.start', bracket: '@open' },
+				{ token: 'tag.tag-$2', bracket: '@open', next: '@tag.$2' }]],
+
+			// Standard closing tag
+			[/(<\/)(@qualifiedName)(\s*)(>)/, [
+				{ token: 'delimiter.end', bracket: '@open' },
+				{ token: 'tag.tag-$2', bracket: '@close' },
+				'',
+				{ token: 'delimiter.end', bracket: '@close' }]],
+
+			// Meta tags - instruction
+			[/(<\?)(@qualifiedName)/, [
+				{ token: 'delimiter.start', bracket: '@open' },
+				{ token: 'metatag.instruction', next: '@tag' }]],
+
+			// Meta tags - declaration
+			[/(<\!)(@qualifiedName)/, [
+				{ token: 'delimiter.start', bracket: '@open' },
+				{ token: 'metatag.declaration', next: '@tag' }]],
+
+			// CDATA
+			[/<\!\[CDATA\[/, { token: 'delimiter.cdata', bracket: '@open', next: '@cdata' }],
+
+			[/&\w+;/, 'string.escape'],
+		],
+
+		cdata: [
+			[/[^\]]+/, ''],
+			[/\]\]>/, { token: 'delimiter.cdata', bracket: '@close', next: '@pop' }],
+			[/\]/, '']
+		],
+
+		tag: [
+			[/[ \t\r\n]+/, '' ],
+			[/(@qualifiedName)(\s*=\s*)("[^"]*"|'[^']*')/, ['attribute.name', '', 'attribute.value']],
+			[/(@qualifiedName)(\s*=\s*)("[^">?\/]*|'[^'>?\/]*)(?=[\?\/]\>)/, ['attribute.name', '', 'attribute.value']],
+			[/(@qualifiedName)(\s*=\s*)("[^">]*|'[^'>]*)/, ['attribute.name', '', 'attribute.value']],
+			[/@qualifiedName/, 'attribute.name'],
+			[/\?>/, { token: 'delimiter.start', bracket: '@close', next: '@pop' }],
+			[/(\/)(>)/, [
+				{ token: 'tag.tag-$S2', bracket: '@close' },
+				{ token: 'delimiter.start', bracket: '@close', next: '@pop' }]],
+			[/>/, { token: 'delimiter.start', bracket: '@close', next: '@pop' }],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/<!--/, { token: 'comment', bracket: '@open', next: '@comment' }]
+		],
+
+		comment: [
+			[/[^<\-]+/, 'comment.content' ],
+			[/-->/,  { token: 'comment', bracket: '@close', next: '@pop' } ],
+			[/<!--/, 'comment.content.invalid'],
+			[/[<\-]/, 'comment.content' ]
+		],
+	},
+};

test/all.js

@@ -0,0 +1,48 @@
+var requirejs = require("requirejs");
+var jsdom = require('jsdom-no-contextify');
+
+requirejs.config({
+	baseUrl: '',
+	paths: {
+		'vs': 'node_modules/monaco-editor-core/dev/vs'
+	},
+	nodeRequire: require
+});
+
+global.document = jsdom.jsdom('<!doctype html><html><body></body></html>');
+global.document.queryCommandSupported = function() {};
+global.self = global.window = global.document.parentWindow;
+global.navigator = global.window.navigator;
+global.window.require = requirejs;
+
+function MyWorker() {}
+MyWorker.prototype.postMessage = function() {};
+global.Worker = MyWorker;
+
+requirejs([
+	'vs/editor/editor.main'
+], function() {
+	requirejs([
+		'out/test/bat.test',
+		'out/test/coffee.test',
+		'out/test/cpp.test',
+		'out/test/csharp.test',
+		'out/test/dockerfile.test',
+		'out/test/fsharp.test',
+		'out/test/go.test',
+		'out/test/jade.test',
+		'out/test/java.test',
+		'out/test/lua.test',
+		'out/test/objective-c.test',
+		'out/test/powershell.test',
+		'out/test/python.test',
+		'out/test/r.test',
+		'out/test/ruby.test',
+		'out/test/swift.test',
+		'out/test/sql.test',
+		'out/test/vb.test',
+		'out/test/xml.test',
+	], function() {
+		run(); // We can launch the tests!
+	});
+});

test/assert.d.ts

@@ -0,0 +1,44 @@
+declare module "assert" {
+    function internal (value: any, message?: string): void;
+    namespace internal {
+        export class AssertionError implements Error {
+            name: string;
+            message: string;
+            actual: any;
+            expected: any;
+            operator: string;
+            generatedMessage: boolean;
+
+            constructor(options?: {message?: string; actual?: any; expected?: any;
+                                  operator?: string; stackStartFunction?: Function});
+        }
+
+        export function fail(actual?: any, expected?: any, message?: string, operator?: string): void;
+        export function ok(value: any, message?: string): void;
+        export function equal(actual: any, expected: any, message?: string): void;
+        export function notEqual(actual: any, expected: any, message?: string): void;
+        export function deepEqual(actual: any, expected: any, message?: string): void;
+        export function notDeepEqual(acutal: any, expected: any, message?: string): void;
+        export function strictEqual(actual: any, expected: any, message?: string): void;
+        export function notStrictEqual(actual: any, expected: any, message?: string): void;
+        export function deepStrictEqual(actual: any, expected: any, message?: string): void;
+        export function notDeepStrictEqual(actual: any, expected: any, message?: string): void;
+        export var throws: {
+            (block: Function, message?: string): void;
+            (block: Function, error: Function, message?: string): void;
+            (block: Function, error: RegExp, message?: string): void;
+            (block: Function, error: (err: any) => boolean, message?: string): void;
+        };
+
+        export var doesNotThrow: {
+            (block: Function, message?: string): void;
+            (block: Function, error: Function, message?: string): void;
+            (block: Function, error: RegExp, message?: string): void;
+            (block: Function, error: (err: any) => boolean, message?: string): void;
+        };
+
+        export function ifError(value: any): void;
+    }
+
+    export = internal;
+}

test/bat.test.ts

@@ -0,0 +1,332 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('bat', [
+	// support.functions
+	[{
+	line: '@echo off title Selfhost',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.bat' },
+		{ startIndex: 1, type: 'support.function.echo.bat' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 10, type: 'support.function.title.bat' },
+		{ startIndex: 15, type: '' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: 'REM',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}],
+
+	[{
+	line: '    REM a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.bat' }
+	]}],
+
+	[{
+	line: 'REM a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}],
+
+	[{
+	line: 'REMnot a comment',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	// constant.numerics
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' }
+	]}],
+
+	[{
+	line: '0.0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.hex.bat' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '1.72e-3',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 1, type: 'punctuation.bat' },
+		{ startIndex: 2, type: 'constant.numeric.bat' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 3, type: 'punctuation.bat' },
+		{ startIndex: 4, type: 'constant.numeric.bat' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'punctuation.bat' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'constant.numeric.bat' }
+	]}],
+
+	// Strings
+	[{
+	line: 'set s = "string"',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.set.bat' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'punctuation.bat' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'string.bat' }
+	]}],
+
+	[{
+	line: '"use strict";',
+	tokens: [
+		{ startIndex: 0, type: 'string.bat' },
+		{ startIndex: 12, type: 'punctuation.bat' }
+	]}],
+
+	// Tags
+	[{
+	line: 'setlocal endlocal',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	[{
+	line: 'setlocal ENDLOCAL',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	[{
+	line: 'SETLOCAL endlocal',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	[{
+	line: 'setlocal setlocal endlocal',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	// Monarch generated
+	[{
+	line: 'rem asdf',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'REM',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'REMOVED not a comment really',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'support.function.not.bat' },
+		{ startIndex: 11, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'echo cool',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.echo.bat' },
+		{ startIndex: 4, type: '' }
+	]}, {
+	line: '@echo off',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.bat' },
+		{ startIndex: 1, type: 'support.function.echo.bat' },
+		{ startIndex: 5, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'setlocAL',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' }
+	]}, {
+	line: '	asdf',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '	asdf',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: 'endLocaL',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'call',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.call.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: ':MyLabel',
+	tokens: [
+		{ startIndex: 0, type: 'metatag.bat' }
+	]}, {
+	line: 'some command',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '%sdfsdf% ',
+	tokens: [
+		{ startIndex: 0, type: 'variable.bat' },
+		{ startIndex: 8, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'this is "a string %sdf% asdf"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'string.bat' },
+		{ startIndex: 18, type: 'variable.bat' },
+		{ startIndex: 23, type: 'string.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'FOR %%A IN (1 2 3) DO (',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.for.bat' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.bat' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 11, type: 'punctuation.parenthesis.bat' },
+		{ startIndex: 12, type: 'constant.numeric.bat' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'constant.numeric.bat' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'constant.numeric.bat' },
+		{ startIndex: 17, type: 'punctuation.parenthesis.bat' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 22, type: 'punctuation.parenthesis.bat' }
+	]}, {
+	line: '	SET VAR1=%VAR1%%%A',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'support.function.set.bat' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 9, type: 'punctuation.bat' },
+		{ startIndex: 10, type: 'variable.bat' }
+	]}, {
+	line: '	SET VAR2=%VAR2%%%A',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'support.function.set.bat' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 9, type: 'punctuation.bat' },
+		{ startIndex: 10, type: 'variable.bat' }
+	]}, {
+	line: '	use \'string %%a asdf asdf\'',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'string.bat' },
+		{ startIndex: 13, type: 'variable.bat' },
+		{ startIndex: 16, type: 'string.bat' }
+	]}, {
+	line: '	non terminated "string %%aaa sdf',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 16, type: 'string.bat' },
+		{ startIndex: 24, type: 'variable.bat' },
+		{ startIndex: 29, type: 'string.bat' }
+	]}, {
+	line: '	this shold NOT BE red',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'support.function.not.bat' },
+		{ startIndex: 15, type: '' }
+	]}, {
+	line: ')',
+	tokens: [
+		{ startIndex: 0, type: 'punctuation.parenthesis.bat' }
+	]}]
+]);

test/coffee.test.ts

@@ -0,0 +1,1983 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('coffeescript', [
+	// Comments
+	[{
+	line: '#',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: '    # a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: '# a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: '#sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: 'x = 1 # my comment # is a nice one',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: 'x = 1e #is a exponent number',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.float.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: 'x = 0x1F #is a hex number',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.hex.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'comment.coffee' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'new x = switch()',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.new.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.switch.coffee' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.coffee' }
+	]}],
+
+	[{
+	line: '@test [do]',
+	tokens: [
+		{ startIndex: 0, type: 'variable.predefined.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.square.coffee' },
+		{ startIndex: 7, type: 'keyword.do.coffee' },
+		{ startIndex: 9, type: 'delimiter.square.coffee' }
+	]}],
+
+	[{
+	line: 'this do',
+	tokens: [
+		{ startIndex: 0, type: 'variable.predefined.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'keyword.do.coffee' }
+	]}],
+
+	[{
+	line: '    new    ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.new.coffee' },
+		{ startIndex: 7, type: '' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '### a simple comment ###',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: 'new x = ### a simple comment ### 1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.new.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.coffee' },
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: 'new x = ### comment ### 1 ###',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.new.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.coffee' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'number.coffee' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: 'x = ######s',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.coffee' },
+		{ startIndex: 10, type: '' }
+	]}],
+
+	[{
+	line: 'x = ###/',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.coffee' }
+	]}],
+
+	// Comments - range comment, multi lines
+	[{
+	line: '### a multiline comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'can actually span',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'multiple lines ###',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: 'new x = ### start a comment',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.new.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.coffee' }
+	]}, {
+	line: ' a ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'and end it ### new a = 2;',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.new.coffee' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 21, type: 'delimiter.coffee' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'number.coffee' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	// Block Strings
+	[{
+	line: 'b(\'\'\'asdads\'\'\')',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 2, type: 'string.coffee' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.coffee' }
+	]}],
+
+	[{
+	line: 'foo(""" var i = \'foo\'; """)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 4, type: 'string.coffee' },
+		{ startIndex: 26, type: 'delimiter.parenthesis.coffee' }
+	]}],
+
+	// Strings
+	[{
+	line: 'for a = \'a\';',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.for.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'string.coffee' },
+		{ startIndex: 11, type: '' }
+	]}],
+
+	[{
+	line: '"use strict";',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 12, type: '' }
+	]}],
+
+	[{
+	line: 'b = a + " \'cool\'  "',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'string.coffee' }
+	]}],
+
+	[{
+	line: '"escaping \\"quotes\\" is cool"',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 10, type: 'string.escape.coffee' },
+		{ startIndex: 12, type: 'string.coffee' },
+		{ startIndex: 18, type: 'string.escape.coffee' },
+		{ startIndex: 20, type: 'string.coffee' }
+	]}],
+
+	[{
+	line: '\'\'\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}],
+
+	[{
+	line: '\'\\\'\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 1, type: 'string.escape.coffee' },
+		{ startIndex: 3, type: 'string.coffee' }
+	]}],
+
+	[{
+	line: '\'be careful \\not to escape\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 12, type: 'string.escape.coffee' },
+		{ startIndex: 14, type: 'string.coffee' }
+	]}],
+
+	// Strings - multiline
+	[{
+	line: '\'a multiline string',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: 'second line',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}],
+
+	// Strings - with nested code
+	[{
+	line: '"for a = \'a\'; #{ new } works"',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'keyword.new.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'string.coffee' }
+	]}],
+
+	[{
+	line: '"a comment with nested code #{ 2 / 3 } works"',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.coffee' },
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'delimiter.coffee' },
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'number.coffee' },
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'string.coffee' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: ' 0',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: ' 0 ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'number.coffee' },
+		{ startIndex: 2, type: '' }
+	]}],
+
+	[{
+	line: '0 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: '' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: 'delimiter.coffee' },
+		{ startIndex: 2, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 3, type: 'delimiter.coffee' },
+		{ startIndex: 4, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: '0123',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.coffee' }
+	]}],
+
+	[{
+	line: '01239',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.coffee' }
+	]}],
+
+	[{
+	line: '[1,2,3]',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.square.coffee' },
+		{ startIndex: 1, type: 'number.coffee' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: 'number.coffee' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: 'number.coffee' },
+		{ startIndex: 6, type: 'delimiter.square.coffee' }
+	]}],
+
+	[{
+	line: 'foo(123);',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 8, type: '' }
+	]}],
+
+	[{
+	line: '(a:(b:[]))',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: 'delimiter.square.coffee' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' }
+	]}],
+
+	[{
+	line: 'x = \'[{()}]\'',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.coffee' }
+	]}],
+
+	// Regular Expressions
+	[{
+	line: '#',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: '/ /',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}],
+
+	[{
+	line: '/abc\\/asd/',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}],
+
+	[{
+	line: 'new r = /sweet"regular exp" \\/ cool/;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.new.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'regexp.coffee' },
+		{ startIndex: 36, type: '' }
+	]}],
+
+	[{
+	line: '5 / 3;',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 5, type: '' }
+	]}],
+
+	// Regex - range regex, multi lines
+	[{
+	line: '/// a multiline regex',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}, {
+	line: 'can actually span',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}, {
+	line: 'multiplelines with # comments',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 19, type: 'comment.coffee' }
+	]}, {
+	line: 'multiple lines ///',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}],
+
+	// Regex - multi lines followed by #comment
+	[{
+	line: '///',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}, {
+	line: '#comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}],
+
+	// Advanced regular expressions
+	[{
+	line: '1 / 2; # comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 7, type: 'comment.coffee' }
+	]}],
+
+	[{
+	line: '1 / 2 / x / b;',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 10, type: 'delimiter.coffee' },
+		{ startIndex: 11, type: '' }
+	]}],
+
+	[{
+	line: 'a /ads/ b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' }
+	]}],
+
+	[{
+	line: 'x = /foo/.test(\'\')',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'regexp.coffee' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 15, type: 'string.coffee' },
+		{ startIndex: 17, type: 'delimiter.parenthesis.coffee' }
+	]}],
+
+	[{
+	line: 'x = 1 + f(2 / 3, /foo/)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 9, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 10, type: 'number.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.coffee' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'number.coffee' },
+		{ startIndex: 15, type: 'delimiter.coffee' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'regexp.coffee' },
+		{ startIndex: 22, type: 'delimiter.parenthesis.coffee' }
+	]}],
+
+	[{
+	line: '1/(2/3)/2/3;',
+	tokens: [
+		{ startIndex: 0, type: 'number.coffee' },
+		{ startIndex: 1, type: 'delimiter.coffee' },
+		{ startIndex: 2, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 3, type: 'number.coffee' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: 'number.coffee' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: 'number.coffee' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: 'number.coffee' },
+		{ startIndex: 11, type: '' }
+	]}],
+
+	[{
+	line: '{ key: 123 }',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'number.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.curly.coffee' }
+	]}],
+
+	[{
+	line: '[1,2,3]',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.square.coffee' },
+		{ startIndex: 1, type: 'number.coffee' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: 'number.coffee' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: 'number.coffee' },
+		{ startIndex: 6, type: 'delimiter.square.coffee' }
+	]}],
+
+	[{
+	line: 'foo(123);',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 4, type: 'number.coffee' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 8, type: '' }
+	]}],
+
+	[{
+	line: '{a:{b:[]}}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.coffee' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: 'delimiter.curly.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: 'delimiter.square.coffee' },
+		{ startIndex: 8, type: 'delimiter.curly.coffee' }
+	]}],
+
+	[{
+	line: 'x = \'[{()}]\'',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.coffee' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.coffee' }
+	]}],
+
+	// syntax highligting issue with {} - bug 16176
+	[{
+	line: '"/api/v2/course/#{ $stateParams.courseId }/grading/student/#{$stateParams.studentId}",',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 31, type: 'delimiter.coffee' },
+		{ startIndex: 32, type: '' },
+		{ startIndex: 41, type: 'string.coffee' },
+		{ startIndex: 61, type: '' },
+		{ startIndex: 73, type: 'delimiter.coffee' },
+		{ startIndex: 74, type: '' },
+		{ startIndex: 83, type: 'string.coffee' },
+		{ startIndex: 85, type: 'delimiter.coffee' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: '# Assignment:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'number   = 42',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'number.coffee' }
+	]}, {
+	line: 'opposite = true',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.true.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Conditions:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'number = -42 if opposite',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: 'number.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'keyword.if.coffee' },
+		{ startIndex: 15, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Functions:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'square = (x) -> x * x',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 18, type: 'delimiter.coffee' },
+		{ startIndex: 19, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Arrays:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'list = [1, 2, 3, 4, 5]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.square.coffee' },
+		{ startIndex: 8, type: 'number.coffee' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'number.coffee' },
+		{ startIndex: 12, type: 'delimiter.coffee' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'number.coffee' },
+		{ startIndex: 15, type: 'delimiter.coffee' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'number.coffee' },
+		{ startIndex: 18, type: 'delimiter.coffee' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'number.coffee' },
+		{ startIndex: 21, type: 'delimiter.square.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Objects:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'math =',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' }
+	]}, {
+	line: '  root:   Math.sqrt',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' }
+	]}, {
+	line: '  square: square',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' }
+	]}, {
+	line: '  cube:   (x) -> x * square x',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 19, type: 'delimiter.coffee' },
+		{ startIndex: 20, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Splats:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'race = (winner, runners...) ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 23, type: 'delimiter.coffee' },
+		{ startIndex: 26, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'delimiter.coffee' }
+	]}, {
+	line: '  print winner, runners',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Existence:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'alert "I knew it!" if elvis?',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'string.coffee' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.if.coffee' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 27, type: 'delimiter.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Array comprehensions:',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'cubes = (math.cube num for num in list)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 13, type: 'delimiter.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 23, type: 'keyword.for.coffee' },
+		{ startIndex: 26, type: '' },
+		{ startIndex: 31, type: 'keyword.in.coffee' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 38, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'fill = (container, liquid = "coffee") ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 17, type: 'delimiter.coffee' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 26, type: 'delimiter.coffee' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'string.coffee' },
+		{ startIndex: 36, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'delimiter.coffee' }
+	]}, {
+	line: '  "Filling the #{container} with #{liquid}..."',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'string.coffee' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 26, type: 'string.coffee' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 41, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ong = ["do", "re", "mi", "fa", "so"]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.square.coffee' },
+		{ startIndex: 7, type: 'string.coffee' },
+		{ startIndex: 11, type: 'delimiter.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'string.coffee' },
+		{ startIndex: 17, type: 'delimiter.coffee' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'string.coffee' },
+		{ startIndex: 23, type: 'delimiter.coffee' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'string.coffee' },
+		{ startIndex: 29, type: 'delimiter.coffee' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'string.coffee' },
+		{ startIndex: 35, type: 'delimiter.square.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'singers = {Jagger: "Rock", Elvis: "Roll"}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.curly.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 17, type: 'delimiter.coffee' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'string.coffee' },
+		{ startIndex: 25, type: 'delimiter.coffee' },
+		{ startIndex: 26, type: '' },
+		{ startIndex: 32, type: 'delimiter.coffee' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 34, type: 'string.coffee' },
+		{ startIndex: 40, type: 'delimiter.curly.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'bitlist = [',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.square.coffee' }
+	]}, {
+	line: '  1, 0, 1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'number.coffee' },
+		{ startIndex: 3, type: 'delimiter.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.coffee' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.coffee' }
+	]}, {
+	line: '  0, 0, 1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'number.coffee' },
+		{ startIndex: 3, type: 'delimiter.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.coffee' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.coffee' }
+	]}, {
+	line: '  1, 1, 0',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'number.coffee' },
+		{ startIndex: 3, type: 'delimiter.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.coffee' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.coffee' }
+	]}, {
+	line: ']',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.square.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'kids =',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' }
+	]}, {
+	line: '  brother:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' }
+	]}, {
+	line: '    name: "Max"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'string.coffee' }
+	]}, {
+	line: '    age:  11',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 10, type: 'number.coffee' }
+	]}, {
+	line: '  sister:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' }
+	]}, {
+	line: '    name: "Ida"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'string.coffee' }
+	]}, {
+	line: '    age:  9',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 10, type: 'number.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '$(\'.account\').attr class: \'active\'',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 2, type: 'string.coffee' },
+		{ startIndex: 12, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 13, type: 'delimiter.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 19, type: 'keyword.class.coffee' },
+		{ startIndex: 24, type: 'delimiter.coffee' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.coffee' }
+	]}, {
+	line: 'log object.class',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'delimiter.coffee' },
+		{ startIndex: 11, type: 'keyword.class.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'outer = 1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.coffee' }
+	]}, {
+	line: 'changeNumbers = ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.coffee' }
+	]}, {
+	line: 'inner = -1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: 'number.coffee' }
+	]}, {
+	line: 'outer = 10',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.coffee' }
+	]}, {
+	line: 'inner = changeNumbers()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 21, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'mood = greatlyImproved if singing',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 23, type: 'keyword.if.coffee' },
+		{ startIndex: 25, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'if happy and knowsIt',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.if.coffee' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 9, type: 'keyword.and.coffee' },
+		{ startIndex: 12, type: '' }
+	]}, {
+	line: '  clapsHands()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '  chaChaCha()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: 'else',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.else.coffee' }
+	]}, {
+	line: '  showIt()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'date = if friday then sue else jill',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.if.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 17, type: 'keyword.then.coffee' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 26, type: 'keyword.else.coffee' },
+		{ startIndex: 30, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'options or= defaults',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.or.coffee' },
+		{ startIndex: 10, type: 'delimiter.coffee' },
+		{ startIndex: 11, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Eat lunch.',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'eat food for food in [\'toast\', \'cheese\', \'wine\']',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'keyword.for.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 18, type: 'keyword.in.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.square.coffee' },
+		{ startIndex: 22, type: 'string.coffee' },
+		{ startIndex: 29, type: 'delimiter.coffee' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'string.coffee' },
+		{ startIndex: 39, type: 'delimiter.coffee' },
+		{ startIndex: 40, type: '' },
+		{ startIndex: 41, type: 'string.coffee' },
+		{ startIndex: 47, type: 'delimiter.square.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Fine five course dining.',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'courses = [\'greens\', \'caviar\', \'truffles\', \'roast\', \'cake\']',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.square.coffee' },
+		{ startIndex: 11, type: 'string.coffee' },
+		{ startIndex: 19, type: 'delimiter.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'string.coffee' },
+		{ startIndex: 29, type: 'delimiter.coffee' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'string.coffee' },
+		{ startIndex: 41, type: 'delimiter.coffee' },
+		{ startIndex: 42, type: '' },
+		{ startIndex: 43, type: 'string.coffee' },
+		{ startIndex: 50, type: 'delimiter.coffee' },
+		{ startIndex: 51, type: '' },
+		{ startIndex: 52, type: 'string.coffee' },
+		{ startIndex: 58, type: 'delimiter.square.coffee' }
+	]}, {
+	line: 'menu i + 1, dish for dish, i in courses',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'number.coffee' },
+		{ startIndex: 10, type: 'delimiter.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 17, type: 'keyword.for.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 25, type: 'delimiter.coffee' },
+		{ startIndex: 26, type: '' },
+		{ startIndex: 29, type: 'keyword.in.coffee' },
+		{ startIndex: 31, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Health conscious meal.',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'foods = [\'broccoli\', \'spinach\', \'chocolate\']',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.square.coffee' },
+		{ startIndex: 9, type: 'string.coffee' },
+		{ startIndex: 19, type: 'delimiter.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'string.coffee' },
+		{ startIndex: 30, type: 'delimiter.coffee' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'string.coffee' },
+		{ startIndex: 43, type: 'delimiter.square.coffee' }
+	]}, {
+	line: 'eat food for food in foods when food isnt \'chocolate\'',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'keyword.for.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 18, type: 'keyword.in.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 27, type: 'keyword.when.coffee' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 37, type: 'keyword.isnt.coffee' },
+		{ startIndex: 41, type: '' },
+		{ startIndex: 42, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'countdown = (num for num in [10..1])',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'delimiter.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 17, type: 'keyword.for.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 25, type: 'keyword.in.coffee' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'delimiter.square.coffee' },
+		{ startIndex: 29, type: 'number.coffee' },
+		{ startIndex: 31, type: 'delimiter.coffee' },
+		{ startIndex: 33, type: 'number.coffee' },
+		{ startIndex: 34, type: 'delimiter.square.coffee' },
+		{ startIndex: 35, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'yearsOld = max: 10, ida: 9, tim: 11',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'number.coffee' },
+		{ startIndex: 18, type: 'delimiter.coffee' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 23, type: 'delimiter.coffee' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'number.coffee' },
+		{ startIndex: 26, type: 'delimiter.coffee' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 31, type: 'delimiter.coffee' },
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'number.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ages = for child, age of yearsOld',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.for.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 16, type: 'delimiter.coffee' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 22, type: 'keyword.of.coffee' },
+		{ startIndex: 24, type: '' }
+	]}, {
+	line: '  "#{child} is #{age}"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'string.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 10, type: 'string.coffee' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 20, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Econ 101',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'if this.studyingEconomics',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.if.coffee' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'variable.predefined.coffee' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' }
+	]}, {
+	line: '  buy()  while supply > demand',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 9, type: 'keyword.while.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 22, type: 'delimiter.coffee' },
+		{ startIndex: 23, type: '' }
+	]}, {
+	line: '  sell() until supply > demand',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.until.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 22, type: 'delimiter.coffee' },
+		{ startIndex: 23, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Nursery Rhyme',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'num = 6',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'number.coffee' }
+	]}, {
+	line: 'lyrics = while num -= 1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.while.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 19, type: 'delimiter.coffee' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.coffee' }
+	]}, {
+	line: '  "#{num} little monkeys, jumping on the bed.',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'string.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 8, type: 'string.coffee' }
+	]}, {
+	line: '    One fell out and bumped his head."',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: '	',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '# Everything is an Expression (at least, as much as possible)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'grade = (student) ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 16, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'delimiter.coffee' }
+	]}, {
+	line: '  if student.excellentWork',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.if.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 12, type: 'delimiter.coffee' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '    "A+"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'string.coffee' }
+	]}, {
+	line: '  else if student.okayStuff',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.else.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.if.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 17, type: 'delimiter.coffee' },
+		{ startIndex: 18, type: '' }
+	]}, {
+	line: '    if student.triedHard then "B" else "B-"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.if.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 14, type: 'delimiter.coffee' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 25, type: 'keyword.then.coffee' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'string.coffee' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 34, type: 'keyword.else.coffee' },
+		{ startIndex: 38, type: '' },
+		{ startIndex: 39, type: 'string.coffee' }
+	]}, {
+	line: '  else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.else.coffee' }
+	]}, {
+	line: '    "C"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'eldest = if 24 > 21 then "Liz" else "Ike"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.if.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'number.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.coffee' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'number.coffee' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'keyword.then.coffee' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'string.coffee' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'keyword.else.coffee' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 36, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#Classes, Inheritance and Super',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'class Animal',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.class.coffee' },
+		{ startIndex: 5, type: '' }
+	]}, {
+	line: '  constructor: (@name) ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 13, type: 'delimiter.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 16, type: 'variable.predefined.coffee' },
+		{ startIndex: 21, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '  move: (meters) ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.coffee' }
+	]}, {
+	line: '    alert @name + " moved #{meters}m."',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'variable.predefined.coffee' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.coffee' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.coffee' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 34, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'class Snake extends Animal',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.class.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 12, type: 'keyword.extends.coffee' },
+		{ startIndex: 19, type: '' }
+	]}, {
+	line: '  move: ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' }
+	]}, {
+	line: '    alert "Slithering..."',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'string.coffee' }
+	]}, {
+	line: '    super 5',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.super.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'number.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'class Horse extends Animal',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.class.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 12, type: 'keyword.extends.coffee' },
+		{ startIndex: 19, type: '' }
+	]}, {
+	line: '  move: ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' }
+	]}, {
+	line: '    alert "Galloping..."',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'string.coffee' }
+	]}, {
+	line: '    super 45',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.super.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'number.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'sam = new Snake "Sammy the Python"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'keyword.new.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 16, type: 'string.coffee' }
+	]}, {
+	line: 'tom = new Horse "Tommy the Palomino"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.coffee' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'keyword.new.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 16, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'sam.move()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: 'tom.move()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.coffee' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#Function binding',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'Account = (customer, cart) ->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 19, type: 'delimiter.coffee' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 25, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 26, type: '' },
+		{ startIndex: 27, type: 'delimiter.coffee' }
+	]}, {
+	line: '  @customer = customer',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.predefined.coffee' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.coffee' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '  @cart = cart',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.predefined.coffee' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.coffee' },
+		{ startIndex: 9, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '  $(\'.shopping_cart\').bind \'click\', (event) =>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 4, type: 'string.coffee' },
+		{ startIndex: 20, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 21, type: 'delimiter.coffee' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 27, type: 'string.coffee' },
+		{ startIndex: 34, type: 'delimiter.coffee' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 36, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 42, type: 'delimiter.parenthesis.coffee' },
+		{ startIndex: 43, type: '' },
+		{ startIndex: 44, type: 'delimiter.coffee' }
+	]}, {
+	line: '    @customer.purchase @cart',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'variable.predefined.coffee' },
+		{ startIndex: 13, type: 'delimiter.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 23, type: 'variable.predefined.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#Switch/When/Else	',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'switch day',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.switch.coffee' },
+		{ startIndex: 6, type: '' }
+	]}, {
+	line: '  when "Mon" then go work',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.when.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'string.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'keyword.then.coffee' },
+		{ startIndex: 17, type: '' }
+	]}, {
+	line: '  when "Tue" then go relax',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.when.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'string.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'keyword.then.coffee' },
+		{ startIndex: 17, type: '' }
+	]}, {
+	line: '  when "Thu" then go iceFishing',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.when.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'string.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'keyword.then.coffee' },
+		{ startIndex: 17, type: '' }
+	]}, {
+	line: '  when "Fri", "Sat"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.when.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'string.coffee' },
+		{ startIndex: 12, type: 'delimiter.coffee' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'string.coffee' }
+	]}, {
+	line: '    if day is bingoDay',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.if.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 11, type: 'keyword.is.coffee' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '      go bingo',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '      go dancing',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '  when "Sun" then go church',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.when.coffee' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'string.coffee' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'keyword.then.coffee' },
+		{ startIndex: 17, type: '' }
+	]}, {
+	line: '  else go work',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.else.coffee' },
+		{ startIndex: 6, type: '' }
+	]}, {
+	line: ' ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '#Try/Catch/Finally',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'try',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.try.coffee' }
+	]}, {
+	line: '  allHellBreaksLoose()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 20, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '  catsAndDogsLivingTogether()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 27, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: 'catch error',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.catch.coffee' },
+		{ startIndex: 5, type: '' }
+	]}, {
+	line: '  print error',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: 'finally',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.finally.coffee' }
+	]}, {
+	line: '  cleanUp()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.parenthesis.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#String Interpolation and Block Comments',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'author = "Wittgenstein"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.coffee' }
+	]}, {
+	line: 'quote  = "A picture is a fact. -- #{ author }"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 7, type: 'delimiter.coffee' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.coffee' },
+		{ startIndex: 36, type: '' },
+		{ startIndex: 44, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'sentence = "#{ 22 / 7 } is a decent approximation of p"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.coffee' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'number.coffee' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'delimiter.coffee' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'number.coffee' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'mobyDick = "Call me Ishmael. Some years ago --',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.coffee' }
+	]}, {
+	line: ' never mind how long precisely -- having little',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: ' or no money in my purse, and nothing particular',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: ' to interest me on shore, I thought I would sail',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: ' about a little and see the watery part of the',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: ' world..."',
+	tokens: [
+		{ startIndex: 0, type: 'string.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#Extended Regular Expressions',
+	tokens: [
+		{ startIndex: 0, type: 'comment.coffee' }
+	]}, {
+	line: 'OPERATOR = /// ^ (',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 9, type: 'delimiter.coffee' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'regexp.coffee' }
+	]}, {
+	line: '  ?: [-=]>             # function',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: '   | [-+*/%<>&|^!?=]=  # compound assign / compare',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: '   | >>>=?             # zero-fill right shift',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: '   | ([-+:])\\1         # doubles',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: '   | ([&|<>])\\2=?      # logic / shift',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: '   | \\?\\.              # soak access',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: '   | \\.{2,3}           # range or splat',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' },
+		{ startIndex: 23, type: 'comment.coffee' }
+	]}, {
+	line: ') ///',
+	tokens: [
+		{ startIndex: 0, type: 'regexp.coffee' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}]
+]);

test/cpp.test.ts

@@ -0,0 +1,666 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('cpp', [
+	// Keywords
+	[{
+	line: 'int _tmain(int argc, _TCHAR* argv[])',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 11, type: 'keyword.int.cpp' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'identifier.cpp' },
+		{ startIndex: 19, type: 'delimiter.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'identifier.cpp' },
+		{ startIndex: 27, type: 'delimiter.cpp' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'identifier.cpp' },
+		{ startIndex: 33, type: 'delimiter.square.cpp' },
+		{ startIndex: 35, type: 'delimiter.parenthesis.cpp' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.cpp' },
+		{ startIndex: 1, type: 'identifier.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cpp' }
+	]}],
+
+	[{
+	line: '/* //*/ a',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cpp' },
+		{ startIndex: 5, type: 'delimiter.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: 'int x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.cpp' },
+		{ startIndex: 9, type: 'delimiter.cpp' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.cpp' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: 'int x = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cpp' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.cpp' },
+		{ startIndex: 32, type: 'delimiter.cpp' }
+	]}],
+
+	[{
+	line: 'int x = /* comment */ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cpp' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.cpp' },
+		{ startIndex: 23, type: 'delimiter.cpp' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cpp' },
+		{ startIndex: 8, type: 'delimiter.cpp' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cpp' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '12l',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '34U',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '55LL',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '34ul',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '55llU',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '5\'5llU',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '100\'000\'000',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '0x100\'aafllU',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.cpp' }
+	]}],
+
+	[{
+	line: '0342\'325',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.cpp' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.cpp' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5l',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3l',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3l',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 1, type: 'delimiter.cpp' },
+		{ startIndex: 2, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 3, type: 'delimiter.cpp' },
+		{ startIndex: 4, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cpp' }
+	]}],
+
+	// Monarch Generated
+	[{
+	line: '#include<iostream>',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 8, type: 'delimiter.angle.cpp' },
+		{ startIndex: 9, type: 'identifier.cpp' },
+		{ startIndex: 17, type: 'delimiter.angle.cpp' }
+	]}, {
+	line: '#include "/path/to/my/file.h"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#ifdef VAR',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' }
+	]}, {
+	line: '#define SUM(A,B) (A) + (B)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 12, type: 'identifier.cpp' },
+		{ startIndex: 13, type: 'delimiter.cpp' },
+		{ startIndex: 14, type: 'identifier.cpp' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 18, type: 'identifier.cpp' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.cpp' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 24, type: 'identifier.cpp' },
+		{ startIndex: 25, type: 'delimiter.parenthesis.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'int main(int argc, char** argv)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 9, type: 'keyword.int.cpp' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.cpp' },
+		{ startIndex: 17, type: 'delimiter.cpp' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.char.cpp' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 26, type: 'identifier.cpp' },
+		{ startIndex: 30, type: 'delimiter.parenthesis.cpp' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '	return 0;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.return.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.cpp' },
+		{ startIndex: 9, type: 'delimiter.cpp' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'namespace TestSpace',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cpp' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '	using Asdf.CDE;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.using.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 11, type: 'delimiter.cpp' },
+		{ startIndex: 12, type: 'identifier.cpp' },
+		{ startIndex: 15, type: 'delimiter.cpp' }
+	]}, {
+	line: '	template <typename T>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.template.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.angle.cpp' },
+		{ startIndex: 11, type: 'keyword.typename.cpp' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.cpp' },
+		{ startIndex: 21, type: 'delimiter.angle.cpp' }
+	]}, {
+	line: '	class CoolClass : protected BaseClass',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.class.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.cpp' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.protected.cpp' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'identifier.cpp' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '		private:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.private.cpp' },
+		{ startIndex: 9, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		static T field;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.static.cpp' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.cpp' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'identifier.cpp' },
+		{ startIndex: 16, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		public:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.public.cpp' },
+		{ startIndex: 8, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		[[deprecated]]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'annotation.cpp' }
+	]}, {
+	line: '		foo method() const override',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.cpp' },
+		{ startIndex: 12, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.const.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'keyword.override.cpp' }
+	]}, {
+	line: '		{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '			auto s = new Bar();',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.auto.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.cpp' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.new.cpp' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.cpp' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 21, type: 'delimiter.cpp' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '			if (s.field) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.if.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 8, type: 'delimiter.cpp' },
+		{ startIndex: 9, type: 'identifier.cpp' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '				for(const auto & b : s.field) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.for.cpp' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 8, type: 'keyword.const.cpp' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.auto.cpp' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'delimiter.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'identifier.cpp' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.cpp' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'identifier.cpp' },
+		{ startIndex: 26, type: 'delimiter.cpp' },
+		{ startIndex: 27, type: 'identifier.cpp' },
+		{ startIndex: 32, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 34, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '					break;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'keyword.break.cpp' },
+		{ startIndex: 10, type: 'delimiter.cpp' }
+	]}, {
+	line: '				}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '			}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '		}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		std::string s = "hello wordld\\n";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'identifier.cpp' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.cpp' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.cpp' },
+		{ startIndex: 31, type: 'string.escape.cpp' },
+		{ startIndex: 33, type: 'string.cpp' },
+		{ startIndex: 34, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		int number = 123\'123\'123Ull;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.int.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.cpp' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'delimiter.cpp' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'number.cpp' },
+		{ startIndex: 29, type: 'delimiter.cpp' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#endif',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' }
+	]}]
+]);

test/csharp.test.ts

@@ -0,0 +1,744 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('csharp', [
+
+	// Generated from sample
+	[{
+	line: 'using System;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Collections.Generic;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 24, type: 'delimiter.cs' },
+		{ startIndex: 25, type: 'namespace.cs' },
+		{ startIndex: 32, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Diagnostics;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 24, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Linq;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 17, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Text;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 17, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Threading.Tasks;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 22, type: 'delimiter.cs' },
+		{ startIndex: 23, type: 'namespace.cs' },
+		{ startIndex: 28, type: 'delimiter.cs' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'namespace VS',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'namespace.cs' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '	class Program',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.class.cs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cs' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '		static void Main(string[] args)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.static.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.void.cs' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'identifier.cs' },
+		{ startIndex: 18, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 19, type: 'keyword.string.cs' },
+		{ startIndex: 25, type: 'delimiter.square.cs' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'identifier.cs' },
+		{ startIndex: 32, type: 'delimiter.parenthesis.cs' }
+	]}, {
+	line: '		{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			ProcessStartInfo si = new ProcessStartInfo();',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.cs' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.cs' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'keyword.new.cs' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'identifier.cs' },
+		{ startIndex: 45, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 47, type: 'delimiter.cs' }
+	]}, {
+	line: '			float load= 3.2e02f;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.float.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.cs' },
+		{ startIndex: 13, type: 'delimiter.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'number.float.cs' },
+		{ startIndex: 22, type: 'delimiter.cs' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '			si.FileName = @"tools\\\\node.exe";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 5, type: 'delimiter.cs' },
+		{ startIndex: 6, type: 'identifier.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.cs' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'string.quote.cs' },
+		{ startIndex: 19, type: 'string.cs' },
+		{ startIndex: 34, type: 'string.quote.cs' },
+		{ startIndex: 35, type: 'delimiter.cs' }
+	]}, {
+	line: '			si.Arguments = "tools\\\\simpl3server.js";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 5, type: 'delimiter.cs' },
+		{ startIndex: 6, type: 'identifier.cs' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.cs' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.quote.cs' },
+		{ startIndex: 19, type: 'string.cs' },
+		{ startIndex: 24, type: 'string.escape.cs' },
+		{ startIndex: 26, type: 'string.cs' },
+		{ startIndex: 41, type: 'string.quote.cs' },
+		{ startIndex: 42, type: 'delimiter.cs' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '			string someString = $"hello{outside+variable}the string again {{ escaped";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.string.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cs' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.cs' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'string.quote.cs' },
+		{ startIndex: 25, type: 'string.cs' },
+		{ startIndex: 30, type: 'string.quote.cs' },
+		{ startIndex: 31, type: 'identifier.cs' },
+		{ startIndex: 38, type: 'delimiter.cs' },
+		{ startIndex: 39, type: 'identifier.cs' },
+		{ startIndex: 47, type: 'string.quote.cs' },
+		{ startIndex: 48, type: 'string.cs' },
+		{ startIndex: 65, type: 'string.escape.cs' },
+		{ startIndex: 67, type: 'string.cs' },
+		{ startIndex: 75, type: 'string.quote.cs' },
+		{ startIndex: 76, type: 'delimiter.cs' }
+	]}, {
+	line: '			var @string = 5;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.var.cs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.cs' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'number.cs' },
+		{ startIndex: 18, type: 'delimiter.cs' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '			if (x == 4)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.if.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 7, type: 'identifier.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.cs' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'number.cs' },
+		{ startIndex: 13, type: 'delimiter.parenthesis.cs' }
+	]}, {
+	line: '			{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '				for (int i = 4; i<10; i++)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.for.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 9, type: 'keyword.int.cs' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.cs' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'number.cs' },
+		{ startIndex: 18, type: 'delimiter.cs' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.cs' },
+		{ startIndex: 21, type: 'delimiter.angle.cs' },
+		{ startIndex: 22, type: 'number.cs' },
+		{ startIndex: 24, type: 'delimiter.cs' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'identifier.cs' },
+		{ startIndex: 27, type: 'delimiter.cs' },
+		{ startIndex: 29, type: 'delimiter.parenthesis.cs' }
+	]}, {
+	line: '				{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '					var d = i;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'keyword.var.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.cs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.cs' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.cs' },
+		{ startIndex: 14, type: 'delimiter.cs' }
+	]}, {
+	line: '				}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.else.cs' }
+	]}, {
+	line: '			{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '				return;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.return.cs' },
+		{ startIndex: 10, type: 'delimiter.cs' }
+	]}, {
+	line: '			}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '			Process.Start(si);',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 10, type: 'delimiter.cs' },
+		{ startIndex: 11, type: 'identifier.cs' },
+		{ startIndex: 16, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 17, type: 'identifier.cs' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 20, type: 'delimiter.cs' }
+	]}, {
+	line: '		}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#pragma region /MapLayer/*Image* /// ',
+	tokens: [
+		{ startIndex: 0, type: 'namespace.cpp.cs' }
+	]}, {
+	line: 'namespace ShouldNotBeAComment {}',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'namespace.cs' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '#pragma endregion Region_1',
+	tokens: [
+		{ startIndex: 0, type: 'namespace.cpp.cs' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'namespace VS { class Program { static void Main(string[] args) {} } }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'namespace.cs' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'delimiter.curly.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.class.cs' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'identifier.cs' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'delimiter.curly.cs' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'keyword.static.cs' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'keyword.void.cs' },
+		{ startIndex: 42, type: '' },
+		{ startIndex: 43, type: 'identifier.cs' },
+		{ startIndex: 47, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 48, type: 'keyword.string.cs' },
+		{ startIndex: 54, type: 'delimiter.square.cs' },
+		{ startIndex: 56, type: '' },
+		{ startIndex: 57, type: 'identifier.cs' },
+		{ startIndex: 61, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 62, type: '' },
+		{ startIndex: 63, type: 'delimiter.curly.cs' },
+		{ startIndex: 65, type: '' },
+		{ startIndex: 66, type: 'delimiter.curly.cs' },
+		{ startIndex: 67, type: '' },
+		{ startIndex: 68, type: 'delimiter.curly.cs' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.cs' },
+		{ startIndex: 1, type: 'identifier.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cs' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cs' },
+		{ startIndex: 5, type: 'delimiter.cs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: 'var x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.var.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.cs' },
+		{ startIndex: 9, type: 'delimiter.cs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.cs' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: 'var x = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.var.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cs' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.cs' },
+		{ startIndex: 32, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'var x = /* comment */ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.var.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cs' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.cs' },
+		{ startIndex: 23, type: 'delimiter.cs' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cs' },
+		{ startIndex: 8, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cs' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' }
+	]}],
+
+	[{
+	line: '0x',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: 'identifier.cs' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.cs' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: 'delimiter.cs' },
+		{ startIndex: 2, type: 'number.cs' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 3, type: 'delimiter.cs' },
+		{ startIndex: 4, type: 'number.cs' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cs' }
+	]}],
+
+	// Strings
+	[{
+	line: 'x = "string";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 5, type: 'string.cs' },
+		{ startIndex: 11, type: 'string.quote.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = "stri\\"ng";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 5, type: 'string.cs' },
+		{ startIndex: 9, type: 'string.escape.cs' },
+		{ startIndex: 11, type: 'string.cs' },
+		{ startIndex: 13, type: 'string.quote.cs' },
+		{ startIndex: 14, type: 'delimiter.cs' }
+	]}],
+
+	// Verbatim Strings
+	[{
+	line: 'x = @"verbatimstring";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' },
+		{ startIndex: 20, type: 'string.quote.cs' },
+		{ startIndex: 21, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = @"verbatim""string";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' },
+		{ startIndex: 14, type: 'string.escape.cs' },
+		{ startIndex: 16, type: 'string.cs' },
+		{ startIndex: 22, type: 'string.quote.cs' },
+		{ startIndex: 23, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = @"verbatim\\string\\";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' },
+		{ startIndex: 22, type: 'string.quote.cs' },
+		{ startIndex: 23, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = @"verbatim',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' }
+	]},{
+	line: 'string";',
+	tokens: [
+		{ startIndex: 0, type: 'string.cs' },
+		{ startIndex: 6, type: 'string.quote.cs' },
+		{ startIndex: 7, type: 'delimiter.cs' }
+	]}]
+]);

test/dockerfile.test.ts

@@ -0,0 +1,189 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('dockerfile', [
+	// All
+	[{
+	line: 'FROM mono:3.12',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 4, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ENV KRE_FEED https://www.myget.org/F/aspnetvnext/api/v2',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.dockerfile' },
+		{ startIndex: 12, type: '' }
+	]}, {
+	line: 'ENV KRE_USER_HOME /opt/kre',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.dockerfile' },
+		{ startIndex: 17, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'RUN apt-get -qq update && apt-get -qqy install unzip ',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ONBUILD RUN curl -sSL https://raw.githubusercontent.com/aspnet/Home/dev/kvminstall.sh | sh',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.dockerfile' },
+		{ startIndex: 11, type: '' }
+	]}, {
+	line: 'ONBUILD RUN bash -c "source $KRE_USER_HOME/kvm/kvm.sh \\',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.dockerfile' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 20, type: 'string.dockerfile' },
+		{ startIndex: 28, type: 'variable.dockerfile' },
+		{ startIndex: 42, type: 'string.dockerfile' }
+	]}, {
+	line: '    && kvm install latest -a default \\',
+	tokens: [
+		{ startIndex: 0, type: 'string.dockerfile' }
+	]}, {
+	line: '    && kvm alias default | xargs -i ln -s $KRE_USER_HOME/packages/{} $KRE_USER_HOME/packages/default"',
+	tokens: [
+		{ startIndex: 0, type: 'string.dockerfile' },
+		{ startIndex: 42, type: 'variable.dockerfile' },
+		{ startIndex: 56, type: 'string.dockerfile' },
+		{ startIndex: 69, type: 'variable.dockerfile' },
+		{ startIndex: 83, type: 'string.dockerfile' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Install libuv for Kestrel from source code (binary is not in wheezy and one in jessie is still too old)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.dockerfile' }
+	]}, {
+	line: 'RUN apt-get -qqy install \\',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: '    autoconf \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    automake \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    build-essential \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    libtool ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: 'RUN LIBUV_VERSION=1.0.0-rc2 \\',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: '    && curl -sSL https://github.com/joyent/libuv/archive/v${LIBUV_VERSION}.tar.gz | tar zxfv - -C /usr/local/src \\',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 58, type: 'variable.dockerfile' },
+		{ startIndex: 74, type: '' }
+	]}, {
+	line: '    && cd /usr/local/src/libuv-$LIBUV_VERSION \\',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 31, type: 'variable.dockerfile' },
+		{ startIndex: 45, type: '' }
+	]}, {
+	line: '    && sh autogen.sh && ./configure && make && make install \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    && rm -rf /usr/local/src/libuv-$LIBUV_VERSION \\',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 35, type: 'variable.dockerfile' },
+		{ startIndex: 49, type: '' }
+	]}, {
+	line: '    && ldconfig',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ENV PATH $PATH:$KRE_USER_HOME/packages/default/bin',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.dockerfile' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'variable.dockerfile' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'variable.dockerfile' },
+		{ startIndex: 29, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Extra things to test',
+	tokens: [
+		{ startIndex: 0, type: 'comment.dockerfile' }
+	]}, {
+	line: 'RUN echo "string at end"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 9, type: 'string.dockerfile' }
+	]}, {
+	line: 'RUN echo must work \'some str\' and some more',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 19, type: 'string.dockerfile' },
+		{ startIndex: 29, type: '' }
+	]}, {
+	line: 'RUN echo hi this is # not a comment',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: 'RUN echo \'String with ${VAR} and another $one here\'',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 9, type: 'string.dockerfile' },
+		{ startIndex: 22, type: 'variable.dockerfile' },
+		{ startIndex: 28, type: 'string.dockerfile' },
+		{ startIndex: 41, type: 'variable.dockerfile' },
+		{ startIndex: 45, type: 'string.dockerfile' }
+	]}]
+]);

test/fsharp.test.ts

@@ -0,0 +1,392 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('fsharp', [
+	// comments - single line
+	[{
+	line: '// one line comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.fs' },
+		{ startIndex: 1, type: 'identifier.fs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.fs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: '(/*almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.parenthesis.fs' },
+		{ startIndex: 1, type: 'delimiter.fs' },
+		{ startIndex: 3, type: 'identifier.fs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.fs' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: '1 / 2; (* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.fs' },
+		{ startIndex: 5, type: 'delimiter.fs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: 'let x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.let.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.fs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.fs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.fs' },
+		{ startIndex: 9, type: 'delimiter.fs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.fs' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'namespace Application1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.fs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: 'type MyType',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.type.fs' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: 'module App =',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.module.fs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.fs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.fs' }
+	]}],
+
+	[{
+	line: 'let AppName = "App1"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.let.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.fs' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.fs' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'string.fs' }
+	]}],
+
+	// Comments - range comment
+	[{
+	line: '(* a simple comment *)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: 'let x = (* a simple comment *) 1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.let.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.fs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.fs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.fs' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.fs' }
+	]}],
+
+	[{
+	line: 'x = (**)',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: 'x = (*)',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.fs' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.fs' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5M',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5m',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3M',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3m',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3M',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3m',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 1, type: 'delimiter.fs' },
+		{ startIndex: 2, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 3, type: 'delimiter.fs' },
+		{ startIndex: 4, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0b00000101',
+	tokens: [
+		{ startIndex: 0, type: 'number.bin.fs' }
+	]}],
+
+	[{
+	line: '86y',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0b00000101y',
+	tokens: [
+		{ startIndex: 0, type: 'number.bin.fs' }
+	]}],
+
+	[{
+	line: '86s',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86us',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86l',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86u',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86ul',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0x00002D3Fn',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.fs' }
+	]}],
+
+	[{
+	line: '0x00002D3Fun',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.fs' }
+	]}],
+
+	[{
+	line: '86L',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86UL',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '9999999999999999999999999999I',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0x00002D3FLF',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}]
+]);

test/go.test.ts

@@ -0,0 +1,1167 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('go', [
+	// Tests
+	[{
+	line: '/* Block comment. */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.go' }
+	]}],
+
+	[{
+	line: '/* //*/ a',
+	tokens: [
+		{ startIndex: 0, type: 'comment.go' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.go' }
+	]}],
+
+	[{
+	line: '// Inline comment.',
+	tokens: [
+		{ startIndex: 0, type: 'comment.go' }
+	]}],
+
+	[{
+	line: '',
+	tokens: [
+
+	]}],
+
+	[{
+	line: 'import {',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.import.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '  "io"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'string.go' }
+	]}],
+
+	[{
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '',
+	tokens: [
+
+	]}],
+
+	[{
+	line: 'type name struct {',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.type.go' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.go' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.struct.go' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '  firstname string',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.string.go' }
+	]}],
+
+	[{
+	line: '  lastname string',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.string.go' }
+	]}],
+
+	[{
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '',
+	tokens: [
+
+	]}],
+
+	[{
+	line: 'func testTypes() {',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.func.go' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.go' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '  a int;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.int.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  b uint;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.uint.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  c uintptr;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.uintptr.go' },
+		{ startIndex: 11, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  d string;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.string.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  e byte;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.byte.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  f rune;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.rune.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  g uint8;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.uint8.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  h uint16;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.uint16.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  i uint32;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.uint32.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  j uint64;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.uint64.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  k int8;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.int8.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  l int16;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.int16.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  m int32;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.int32.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  n int64;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.int64.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  o float32;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.float32.go' },
+		{ startIndex: 11, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  p float64;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.float64.go' },
+		{ startIndex: 11, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  q complex64;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.complex64.go' },
+		{ startIndex: 13, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  r complex128;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'keyword.complex128.go' },
+		{ startIndex: 14, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '',
+	tokens: [
+
+	]}],
+
+	[{
+	line: 'func testOperators() {',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.func.go' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.go' },
+		{ startIndex: 18, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '  ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '  var a;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  var b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '  a + b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a - b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a * b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a / b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a % b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a & b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a | b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a ^ b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a << b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a >> b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a &^ b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a += b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a -= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a *= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a /= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a %= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a &= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a |= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a ^= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a <<= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a >>= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a &^= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a && b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a || b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a <- b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a++;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  b--;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a == b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a < b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.angle.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a > b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.angle.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a = b; ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' },
+		{ startIndex: 8, type: '' }
+	]}],
+
+	[{
+	line: '  !a;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.go' },
+		{ startIndex: 3, type: 'identifier.go' },
+		{ startIndex: 4, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a != b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a <= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a >= b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a := b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a...;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  (a)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 3, type: 'identifier.go' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.go' }
+	]}],
+
+	[{
+	line: '  [a]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.square.go' },
+		{ startIndex: 3, type: 'identifier.go' },
+		{ startIndex: 4, type: 'delimiter.square.go' }
+	]}],
+
+	[{
+	line: '  a.b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: 'delimiter.go' },
+		{ startIndex: 4, type: 'identifier.go' },
+		{ startIndex: 5, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a, b;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: 'delimiter.go' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.go' },
+		{ startIndex: 6, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  a : b; ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.go' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' },
+		{ startIndex: 8, type: '' }
+	]}],
+
+	[{
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '',
+	tokens: [
+
+	]}],
+
+	[{
+	line: 'func keywords() {',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.func.go' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.go' },
+		{ startIndex: 13, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '  ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '  var a;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.go' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  break;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.break.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  switch(a) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.switch.go' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 9, type: 'identifier.go' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '    case 1:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.case.go' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'number.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '      fallthrough;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'keyword.fallthrough.go' },
+		{ startIndex: 17, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '    default:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.default.go' },
+		{ startIndex: 11, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '      break;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'keyword.break.go' },
+		{ startIndex: 11, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  }',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.go' }
+	]}],
+
+	[{
+	line: '  ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '  chan;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.chan.go' },
+		{ startIndex: 6, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  const;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.const.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  continue;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.continue.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  defer;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.defer.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  if (a)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.if.go' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 6, type: 'identifier.go' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.go' }
+	]}],
+
+	[{
+	line: '    return;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.return.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '  else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.else.go' }
+	]}],
+
+	[{
+	line: '    return;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.return.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   for (i = 0; i < 10; i++);',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.for.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 8, type: 'identifier.go' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.go' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'number.go' },
+		{ startIndex: 13, type: 'delimiter.go' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'identifier.go' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.angle.go' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'number.go' },
+		{ startIndex: 21, type: 'delimiter.go' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'identifier.go' },
+		{ startIndex: 24, type: 'delimiter.go' },
+		{ startIndex: 26, type: 'delimiter.parenthesis.go' },
+		{ startIndex: 27, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   go;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.go.go' },
+		{ startIndex: 5, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   goto;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.goto.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   interface;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.interface.go' },
+		{ startIndex: 12, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   map;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.map.go' },
+		{ startIndex: 6, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   package;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.package.go' },
+		{ startIndex: 10, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   range;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.range.go' },
+		{ startIndex: 8, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   return;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.return.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   select;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.select.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   struct;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.struct.go' },
+		{ startIndex: 9, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   type;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.type.go' },
+		{ startIndex: 7, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '   var x = true;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.var.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.go' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.true.go' },
+		{ startIndex: 15, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   var y = false;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.var.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.go' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.false.go' },
+		{ startIndex: 16, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '   var z = nil;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.var.go' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.go' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.go' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.nil.go' },
+		{ startIndex: 14, type: 'delimiter.go' }
+	]}],
+
+	[{
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.go' }
+	]}]
+]);

test/jade.test.ts

@@ -0,0 +1,380 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('jade', [
+	// Tags [Jade]
+	[{
+	line: 'p 5',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 1, type: '' }
+	]}],
+
+	[{
+	line: 'div#container.stuff',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: 'tag.id.jade' },
+		{ startIndex: 13, type: 'tag.class.jade' }
+	]}],
+
+	[{
+	line: 'div.container#stuff',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: 'tag.class.jade' },
+		{ startIndex: 13, type: 'tag.id.jade' }
+	]}],
+
+	[{
+	line: 'div.container#stuff .container',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: 'tag.class.jade' },
+		{ startIndex: 13, type: 'tag.id.jade' },
+		{ startIndex: 19, type: '' }
+	]}],
+
+	[{
+	line: '#tag-id-1',
+	tokens: [
+		{ startIndex: 0, type: 'tag.id.jade' }
+	]}],
+
+	[{
+	line: '.tag-id-1',
+	tokens: [
+		{ startIndex: 0, type: 'tag.class.jade' }
+	]}],
+
+	// Attributes - Single Line [Jade]
+	[{
+	line: 'input(type="checkbox")',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' },
+		{ startIndex: 21, type: 'delimiter.parenthesis.jade' }
+	]}],
+
+	[{
+	line: 'input (type="checkbox")',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: '' }
+	]}],
+
+	[{
+	line: 'input(type="checkbox",name="agreement",checked)',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' },
+		{ startIndex: 21, type: 'attribute.delimiter.jade' },
+		{ startIndex: 22, type: 'attribute.name.jade' },
+		{ startIndex: 26, type: 'delimiter.jade' },
+		{ startIndex: 27, type: 'attribute.value.jade' },
+		{ startIndex: 38, type: 'attribute.delimiter.jade' },
+		{ startIndex: 39, type: 'attribute.name.jade' },
+		{ startIndex: 46, type: 'delimiter.parenthesis.jade' }
+	]}],
+
+	[{
+	line: 'input(type="checkbox"',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' }
+	]}, {
+	line: 'name="agreement"',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' },
+		{ startIndex: 4, type: 'delimiter.jade' },
+		{ startIndex: 5, type: 'attribute.value.jade' }
+	]}, {
+	line: 'checked)',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: 'body',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' }
+	]}],
+
+	// Attributes - MultiLine [Jade]
+	[{
+	line: 'input(type="checkbox"',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' }
+	]}, {
+	line: 'disabled',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' }
+	]}, {
+	line: 'checked)',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: 'body',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' }
+	]}],
+
+	// Interpolation [Jade]
+	[{
+	line: 'p print #{count} lines',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 8, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 10, type: 'interpolation.jade' },
+		{ startIndex: 15, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 16, type: '' }
+	]}],
+
+	[{
+	line: 'p print "#{count}" lines',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 9, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 11, type: 'interpolation.jade' },
+		{ startIndex: 16, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 17, type: '' }
+	]}],
+
+	[{
+	line: '{ key: 123 }',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.jade' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 5, type: 'delimiter.jade' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'number.jade' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.curly.jade' }
+	]}],
+
+	// Comments - Single Line [Jade]
+	[{
+	line: '// html#id1.class1',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}],
+
+	[{
+	line: 'body hello // not a comment 123',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 4, type: '' }
+	]}],
+
+	// Comments - MultiLine [Jade]
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '    should be a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '    should still be a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: 'div should not be a comment',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: '' }
+	]}],
+
+	// Code [Jade]
+	[{
+	line: '- var a = 1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.jade' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 8, type: 'delimiter.jade' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'number.jade' }
+	]}],
+
+	[{
+	line: 'each item in items',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.each.jade' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 10, type: 'keyword.in.jade' },
+		{ startIndex: 12, type: '' }
+	]}],
+
+	[{
+	line: '- var html = "<script></script>"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.jade' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 11, type: 'delimiter.jade' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'string.jade' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: 'doctype 5',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.doctype.jade' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.jade' }
+	]}, {
+	line: 'html(lang="en")',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 5, type: 'attribute.name.jade' },
+		{ startIndex: 9, type: 'delimiter.jade' },
+		{ startIndex: 10, type: 'attribute.value.jade' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: '    head',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'tag.jade' }
+	]}, {
+	line: '        title= pageTitle',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '        script(type=\'text/javascript\')',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 15, type: 'attribute.name.jade' },
+		{ startIndex: 19, type: 'delimiter.jade' },
+		{ startIndex: 20, type: 'attribute.value.jade' },
+		{ startIndex: 37, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: '            if (foo) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'keyword.if.jade' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.curly.jade' }
+	]}, {
+	line: '                bar()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: '            }',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'delimiter.curly.jade' }
+	]}, {
+	line: '    body',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'tag.jade' }
+	]}, {
+	line: '        // Disclaimer: You will need to turn insertSpaces to true in order for the',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '         syntax highlighting to kick in properly (especially for comments)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '            Enjoy :)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '        h1 Jade - node template engine if in',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 10, type: '' }
+	]}, {
+	line: '        p.',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 9, type: 'delimiter.jade' }
+	]}, {
+	line: '          text ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '            text',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '          #container',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '         #container',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '        #container',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.id.jade' }
+	]}, {
+	line: '          if youAreUsingJade',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'keyword.if.jade' },
+		{ startIndex: 12, type: '' }
+	]}, {
+	line: '            p You are amazing',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'tag.jade' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '          else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'keyword.else.jade' }
+	]}, {
+	line: '            p Get on it!',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'tag.jade' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '     p Text can be included in a number of different ways.',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'tag.jade' },
+		{ startIndex: 6, type: '' }
+	]}]
+]);

test/java.test.ts

@@ -0,0 +1,605 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('java', [
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.java' }
+	]}],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[{
+	line: '/* //*/ a',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.java' },
+		{ startIndex: 1, type: 'identifier.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.java' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.java' },
+		{ startIndex: 5, type: 'delimiter.java' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.java' }
+	]}],
+
+	[{
+	line: 'int x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.java' },
+		{ startIndex: 9, type: 'delimiter.java' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.java' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: 'int x = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.java' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.java' },
+		{ startIndex: 32, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: 'int x = /* comment */ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.java' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.java' },
+		{ startIndex: 23, type: 'delimiter.java' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.java' },
+		{ startIndex: 8, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.java' }
+	]}],
+
+	// Comments - range comment, multiple lines
+	[{
+	line: '/* start of multiline comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}, {
+	line: 'a comment between without a star',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}, {
+	line: 'end of multiline comment*/',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: 'int x = /* start a comment',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.java' }
+	]}, {
+	line: ' a ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}, {
+	line: 'and end it */ 2;',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'number.java' },
+		{ startIndex: 15, type: 'delimiter.java' }
+	]}],
+
+	// Java Doc, multiple lines
+	[{
+	line: '/** start of Java Doc',
+	tokens: [
+		{ startIndex: 0, type: 'comment.doc.java' }
+	]}, {
+	line: 'a comment between without a star',
+	tokens: [
+		{ startIndex: 0, type: 'comment.doc.java' }
+	]}, {
+	line: 'end of multiline comment*/',
+	tokens: [
+		{ startIndex: 0, type: 'comment.doc.java' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'package test; class Program { static void main(String[] args) {} } }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.package.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.java' },
+		{ startIndex: 12, type: 'delimiter.java' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.class.java' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.java' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'delimiter.curly.java' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'keyword.static.java' },
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'keyword.void.java' },
+		{ startIndex: 41, type: '' },
+		{ startIndex: 42, type: 'identifier.java' },
+		{ startIndex: 46, type: 'delimiter.parenthesis.java' },
+		{ startIndex: 47, type: 'identifier.java' },
+		{ startIndex: 53, type: 'delimiter.square.java' },
+		{ startIndex: 55, type: '' },
+		{ startIndex: 56, type: 'identifier.java' },
+		{ startIndex: 60, type: 'delimiter.parenthesis.java' },
+		{ startIndex: 61, type: '' },
+		{ startIndex: 62, type: 'delimiter.curly.java' },
+		{ startIndex: 64, type: '' },
+		{ startIndex: 65, type: 'delimiter.curly.java' },
+		{ startIndex: 66, type: '' },
+		{ startIndex: 67, type: 'delimiter.curly.java' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '0.10',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '0x',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.java' }
+	]}],
+
+	[{
+	line: '0x5_2',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.java' }
+	]}],
+
+	[{
+	line: '023L',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' }
+	]}],
+
+	[{
+	line: '0123l',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' }
+	]}],
+
+	[{
+	line: '05_2',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' }
+	]}],
+
+	[{
+	line: '0b1010_0101',
+	tokens: [
+		{ startIndex: 0, type: 'number.binary.java' }
+	]}],
+
+	[{
+	line: '0B001',
+	tokens: [
+		{ startIndex: 0, type: 'number.binary.java' }
+	]}],
+
+	[{
+	line: '10e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '10f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5e-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5E-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72E3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72E3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72E-3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72e3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72e3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72e-3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23L',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '23l',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '0_52',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '5_2',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '5_______2',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '3_.1415F',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '3._1415F',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'delimiter.java' },
+		{ startIndex: 2, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '999_99_9999_L',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 11, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '52_',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 2, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0_x52',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0x_52',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0x52_',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.java' },
+		{ startIndex: 4, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '052_',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' },
+		{ startIndex: 3, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '23.5L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' },
+		{ startIndex: 4, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'delimiter.java' },
+		{ startIndex: 2, type: 'number.java' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 3, type: 'delimiter.java' },
+		{ startIndex: 4, type: 'number.java' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.java' }
+	]}],
+
+	// single line Strings
+	[{
+	line: 'String s = "I\'m a Java String";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.java' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.java' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.java' },
+		{ startIndex: 30, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: 'String s = "concatenated" + " String" ;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.java' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.java' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.java' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'delimiter.java' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'string.java' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: '"quote in a string"',
+	tokens: [
+		{ startIndex: 0, type: 'string.java' }
+	]}],
+
+	[{
+	line: '"escaping \\"quotes\\" is cool"',
+	tokens: [
+		{ startIndex: 0, type: 'string.java' },
+		{ startIndex: 10, type: 'string.escape.java' },
+		{ startIndex: 12, type: 'string.java' },
+		{ startIndex: 18, type: 'string.escape.java' },
+		{ startIndex: 20, type: 'string.java' }
+	]}],
+
+	[{
+	line: '"\\"',
+	tokens: [
+		{ startIndex: 0, type: 'string.invalid.java' }
+	]}],
+
+	// Annotations
+	[{
+	line: '@',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '@Override',
+	tokens: [
+		{ startIndex: 0, type: 'annotation.java' }
+	]}],
+
+	[{
+	line: '@SuppressWarnings(value = "aString")',
+	tokens: [
+		{ startIndex: 0, type: 'annotation.java' },
+		{ startIndex: 17, type: 'delimiter.parenthesis.java' },
+		{ startIndex: 18, type: 'identifier.java' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'delimiter.java' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.java' },
+		{ startIndex: 35, type: 'delimiter.parenthesis.java' }
+	]}],
+
+	[{
+	line: '@ AnnotationWithKeywordAfter private',
+	tokens: [
+		{ startIndex: 0, type: 'annotation.java' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'keyword.private.java' }
+	]}]
+]);
+

test/lua.test.ts

@@ -0,0 +1,76 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('lua', [
+
+	// Keywords
+	[{
+	line: 'local x, y = 1, 10',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.local.lua' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.lua' },
+		{ startIndex: 7, type: 'delimiter.lua' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.lua' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.lua' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'number.lua' },
+		{ startIndex: 14, type: 'delimiter.lua' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'number.lua' }
+	]}],
+
+	[{
+	line: 'foo = "Hello" .. "World"; local foo = foo',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.lua' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.lua' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'string.lua' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'delimiter.lua' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'string.lua' },
+		{ startIndex: 24, type: 'delimiter.lua' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'keyword.local.lua' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'identifier.lua' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 36, type: 'delimiter.lua' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'identifier.lua' }
+	]}],
+
+	// Comments
+	[{
+	line: '--[[ text ]] x',
+	tokens: [
+		{ startIndex: 0, type: 'comment.lua' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.lua' }
+	]}],
+
+	[{
+	line: '--[===[ text ]===] x',
+	tokens: [
+		{ startIndex: 0, type: 'comment.lua' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'identifier.lua' }
+	]}],
+
+	[{
+	line: '--[===[ text ]==] x',
+	tokens: [
+		{ startIndex: 0, type: 'comment.lua' }
+	]}]
+]);

test/mocha.d.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+declare function run(): void;
+
+declare function suite(name: string, fn: (err?)=>void);
+declare function test(name: string, fn: (done?: (err?)=>void)=>void);
+declare function suiteSetup(fn: (done?: (err?)=>void)=>void);
+declare function suiteTeardown(fn: (done?: (err?)=>void)=>void);
+declare function setup(fn: (done?: (err?)=>void)=>void);
+declare function teardown(fn: (done?: (err?)=>void)=>void);

test/mocha.opts

@@ -0,0 +1,3 @@
+--delay
+--ui tdd
+test/all.js

test/objective-c.test.ts

@@ -0,0 +1,291 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('objective-c', [
+	// Keywords
+	[{
+	line: '-(id) initWithParams:(id<anObject>) aHandler withDeviceStateManager:(id<anotherObject>) deviceStateManager',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 2, type: 'keyword.objective-c' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'identifier.objective-c' },
+		{ startIndex: 20, type: 'delimiter.objective-c' },
+		{ startIndex: 21, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 22, type: 'keyword.objective-c' },
+		{ startIndex: 24, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 25, type: 'identifier.objective-c' },
+		{ startIndex: 33, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 34, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 35, type: 'white.objective-c' },
+		{ startIndex: 36, type: 'identifier.objective-c' },
+		{ startIndex: 44, type: 'white.objective-c' },
+		{ startIndex: 45, type: 'identifier.objective-c' },
+		{ startIndex: 67, type: 'delimiter.objective-c' },
+		{ startIndex: 68, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 69, type: 'keyword.objective-c' },
+		{ startIndex: 71, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 72, type: 'identifier.objective-c' },
+		{ startIndex: 85, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 86, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 87, type: 'white.objective-c' },
+		{ startIndex: 88, type: 'identifier.objective-c' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'operator.objective-c' },
+		{ startIndex: 1, type: 'identifier.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'identifier.objective-c' },
+		{ startIndex: 9, type: 'white.objective-c' },
+		{ startIndex: 10, type: 'identifier.objective-c' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'number.objective-c' },
+		{ startIndex: 5, type: 'delimiter.objective-c' },
+		{ startIndex: 6, type: 'white.objective-c' },
+		{ startIndex: 7, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: 'int x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'identifier.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'operator.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'number.objective-c' },
+		{ startIndex: 9, type: 'delimiter.objective-c' },
+		{ startIndex: 10, type: 'white.objective-c' },
+		{ startIndex: 11, type: 'comment.objective-c' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: 'int x = /* embedded comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'identifier.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'operator.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'comment.objective-c' },
+		{ startIndex: 30, type: 'white.objective-c' },
+		{ startIndex: 31, type: 'number.objective-c' },
+		{ startIndex: 32, type: 'delimiter.objective-c' }
+	]}],
+
+	[{
+	line: 'int x = /* comment and syntax error*/ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'identifier.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'operator.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'comment.objective-c' },
+		{ startIndex: 37, type: 'white.objective-c' },
+		{ startIndex: 38, type: 'number.objective-c' },
+		{ startIndex: 39, type: 'delimiter.objective-c' },
+		{ startIndex: 40, type: 'white.objective-c' },
+		{ startIndex: 41, type: 'operator.objective-c' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'comment.objective-c' },
+		{ startIndex: 8, type: 'delimiter.objective-c' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'comment.objective-c' }
+	]}],
+
+	// Non-Alpha Keywords
+	[{
+	line: '#import <GTLT.h>',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 9, type: 'identifier.objective-c' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'identifier.objective-c' },
+		{ startIndex: 15, type: 'delimiter.angle.objective-c' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '0x ',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.objective-c' },
+		{ startIndex: 2, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '0x123 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 4, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5e3 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 6, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5E3 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 6, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5F ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5f ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72E3F ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72E3f ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72e3F ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72e3f ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'operator.objective-c' },
+		{ startIndex: 2, type: 'number.objective-c' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 3, type: 'operator.objective-c' },
+		{ startIndex: 4, type: 'number.objective-c' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'number.objective-c' }
+	]}]
+]);

test/powershell.test.ts

@@ -0,0 +1,737 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('powershell', [
+	// Comments - single line
+	[{
+	line: '#',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '    # a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '# a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '#sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '##still a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '1 / 2 /# comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.ps1' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.ps1' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.ps1' },
+		{ startIndex: 7, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '$x = 1 # my comment # is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.ps1' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '<# a simple comment #>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '$x = <# a simple comment #> 1',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '$yy = <# comment #> 14',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.ps1' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'comment.ps1' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '$x = <##>7',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' },
+		{ startIndex: 9, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '$x = <#<85',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' }
+	]}],
+
+	// Comments - range comment, multiple lines
+	[{
+	line: '<# start of multiline comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'a comment between',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'end of multiline comment#>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '$x = <# start a comment',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' }
+	]}, {
+	line: ' a ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'and end it #> 2',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'number.ps1' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'foreach($i in $b) {if (7) continue}',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.foreach.ps1' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 8, type: 'variable.ps1' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.in.ps1' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'variable.ps1' },
+		{ startIndex: 16, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'delimiter.curly.ps1' },
+		{ startIndex: 19, type: 'keyword.if.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 23, type: 'number.ps1' },
+		{ startIndex: 24, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'keyword.continue.ps1' },
+		{ startIndex: 34, type: 'delimiter.curly.ps1' }
+	]}],
+
+	// Redirect operand
+	[{
+	line: '$i > output1.txt',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 12, type: 'delimiter.ps1' },
+		{ startIndex: 13, type: '' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '0.10',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '0X123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.ps1' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.ps1' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5e-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5E-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 1, type: 'delimiter.ps1' },
+		{ startIndex: 2, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '10 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.ps1' }
+	]}],
+
+	// Strings
+	[{
+	line: '$s = "I am a String"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '\'I am also a ( String\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '$s = "concatenated" + " String"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'delimiter.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '"escaping `"quotes`" is cool"',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 10, type: 'string.escape.ps1' },
+		{ startIndex: 12, type: 'string.ps1' },
+		{ startIndex: 18, type: 'string.escape.ps1' },
+		{ startIndex: 20, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '\'`\'end of the string',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 1, type: 'string.escape.ps1' },
+		{ startIndex: 3, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '@"I am an expandable String"@',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '@\'I am also an expandable String\'@',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '$s = @\'I am also an expandable String\'@',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '$s = @\'I am also an expandable String\'@+7',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '@\'I am a multiline string,',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: 'and this is the middle line,',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: 'and this is NOT the end of the string\'@foreach $i',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: '\'@',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: '${script:foo}',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' }
+	]}, {
+	line: 'foreach $i',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.foreach.ps1' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'variable.ps1' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: '$SelectedObjectNames=@();',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 20, type: 'delimiter.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 24, type: 'delimiter.ps1' }
+	]}, {
+	line: '$XenCenterNodeSelected = 0;',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'number.ps1' },
+		{ startIndex: 26, type: 'delimiter.ps1' }
+	]}, {
+	line: '#the object info array contains hashmaps, each of which represent a parameter set and describe a target in the XenCenter resource list',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'foreach($parameterSet in $ObjInfoArray)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.foreach.ps1' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 8, type: 'variable.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'keyword.in.ps1' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'variable.ps1' },
+		{ startIndex: 38, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '	if ($parameterSet["class"] -eq "blank")',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.if.ps1' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 5, type: 'variable.ps1' },
+		{ startIndex: 18, type: 'delimiter.square.ps1' },
+		{ startIndex: 19, type: 'string.ps1' },
+		{ startIndex: 26, type: 'delimiter.square.ps1' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'delimiter.ps1' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 32, type: 'string.ps1' },
+		{ startIndex: 39, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		#When the XenCenter node is selected a parameter set is created for each of your connected servers with the class and objUuid keys marked as blank',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '		if ($XenCenterNodeSelected)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.if.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 6, type: 'variable.ps1' },
+		{ startIndex: 28, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '		{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '			continue',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.continue.ps1' }
+	]}, {
+	line: '		}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		$XenCenterNodeSelected = 1;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'delimiter.ps1' },
+		{ startIndex: 26, type: '' },
+		{ startIndex: 27, type: 'number.ps1' },
+		{ startIndex: 28, type: 'delimiter.ps1' }
+	]}, {
+	line: '		$SelectedObjectNames += "XenCenter"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.ps1' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '	elseif ($parameterSet["sessionRef"] -eq "null")',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.elseif.ps1' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 9, type: 'variable.ps1' },
+		{ startIndex: 22, type: 'delimiter.square.ps1' },
+		{ startIndex: 23, type: 'string.ps1' },
+		{ startIndex: 35, type: 'delimiter.square.ps1' },
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'delimiter.ps1' },
+		{ startIndex: 38, type: '' },
+		{ startIndex: 41, type: 'string.ps1' },
+		{ startIndex: 47, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		#When a disconnected server is selected there is no session information, we get null for everything except class',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		$SelectedObjectNames += "a disconnected server"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.ps1' }
+	]}, {
+	line: '	else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.else.ps1' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		Connect-XenServer -url $parameterSet["url"] -opaqueref $parameterSet["sessionRef"]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 20, type: 'delimiter.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 25, type: 'variable.ps1' },
+		{ startIndex: 38, type: 'delimiter.square.ps1' },
+		{ startIndex: 39, type: 'string.ps1' },
+		{ startIndex: 44, type: 'delimiter.square.ps1' },
+		{ startIndex: 45, type: '' },
+		{ startIndex: 46, type: 'delimiter.ps1' },
+		{ startIndex: 47, type: '' },
+		{ startIndex: 57, type: 'variable.ps1' },
+		{ startIndex: 70, type: 'delimiter.square.ps1' },
+		{ startIndex: 71, type: 'string.ps1' },
+		{ startIndex: 83, type: 'delimiter.square.ps1' }
+	]}, {
+	line: '		#Use $class to determine which server objects to get',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '		#-properties allows us to filter the results to just include the selected object',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '		$exp = "Get-XenServer:{0} -properties @{{uuid=\'{1}\'}}" -f $parameterSet["class"], $parameterSet["objUuid"]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.ps1' },
+		{ startIndex: 56, type: '' },
+		{ startIndex: 57, type: 'delimiter.ps1' },
+		{ startIndex: 58, type: '' },
+		{ startIndex: 60, type: 'variable.ps1' },
+		{ startIndex: 73, type: 'delimiter.square.ps1' },
+		{ startIndex: 74, type: 'string.ps1' },
+		{ startIndex: 81, type: 'delimiter.square.ps1' },
+		{ startIndex: 82, type: 'delimiter.ps1' },
+		{ startIndex: 83, type: '' },
+		{ startIndex: 84, type: 'variable.ps1' },
+		{ startIndex: 97, type: 'delimiter.square.ps1' },
+		{ startIndex: 98, type: 'string.ps1' },
+		{ startIndex: 107, type: 'delimiter.square.ps1' }
+	]}, {
+	line: '		$obj = Invoke-Expression $exp',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 27, type: 'variable.ps1' }
+	]}, {
+	line: '		$SelectedObjectNames += $obj.name_label;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'variable.ps1' },
+		{ startIndex: 30, type: 'delimiter.ps1' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 41, type: 'delimiter.ps1' }
+	]}, {
+	line: '	} ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' },
+		{ startIndex: 2, type: '' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '$test = "in string var$test"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.ps1' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'string.ps1' },
+		{ startIndex: 22, type: 'variable.ps1' },
+		{ startIndex: 27, type: 'string.ps1' }
+	]}, {
+	line: '$another = \'not a $var\'',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.ps1' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.ps1' }
+	]}, {
+	line: '$third = "a $var and not `$var string"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.ps1' },
+		{ startIndex: 12, type: 'variable.ps1' },
+		{ startIndex: 16, type: 'string.ps1' },
+		{ startIndex: 25, type: 'string.escape.ps1' },
+		{ startIndex: 27, type: 'string.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: ':aLabel',
+	tokens: [
+		{ startIndex: 0, type: 'metatag.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '<#',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '.SYNOPSIS',
+	tokens: [
+		{ startIndex: 0, type: 'comment.keyword.synopsis.ps1' }
+	]}, {
+	line: '  some text',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '  ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '.LINK',
+	tokens: [
+		{ startIndex: 0, type: 'comment.keyword.link.ps1' }
+	]}, {
+	line: '  some more text',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '#>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '$hereString = @"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.ps1' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'string.ps1' }
+	]}, {
+	line: '  a string',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: '  still "@ a string $withVar',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 20, type: 'variable.ps1' }
+	]}, {
+	line: '  still a string `$noVar',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 17, type: 'string.escape.ps1' },
+		{ startIndex: 19, type: 'string.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '"@ still a string',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 2, type: '' }
+	]}]
+]);

test/python.test.ts

@@ -0,0 +1,105 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('python', [
+	// Keywords
+	[{
+	line: 'def func():',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.python' },
+		{ startIndex: 3, type: 'white.python' },
+		{ startIndex: 4, type: 'identifier.python' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.python' },
+		{ startIndex: 10, type: 'delimiter.python' }
+	]}],
+
+	[{
+	line: 'func(str Y3)',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.python' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.python' },
+		{ startIndex: 5, type: 'keyword.python' },
+		{ startIndex: 8, type: 'white.python' },
+		{ startIndex: 9, type: 'identifier.python' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.python' }
+	]}],
+
+	[{
+	line: '@Dec0_rator:',
+	tokens: [
+		{ startIndex: 0, type: 'tag.python' },
+		{ startIndex: 11, type: 'delimiter.python' }
+	]}],
+
+	// Comments
+	[{
+	line: ' # Comments! ## "jfkd" ',
+	tokens: [
+		{ startIndex: 0, type: 'white.python' },
+		{ startIndex: 1, type: 'comment.python' }
+	]}],
+
+	// Strings
+	[{
+	line: '\'s0\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.python' },
+		{ startIndex: 1, type: 'string.python' },
+		{ startIndex: 3, type: 'string.escape.python' }
+	]}],
+
+	[{
+	line: '"\' " "',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.python' },
+		{ startIndex: 1, type: 'string.python' },
+		{ startIndex: 3, type: 'string.escape.python' },
+		{ startIndex: 4, type: 'white.python' },
+		{ startIndex: 5, type: 'string.escape.python' }
+	]}],
+
+	[{
+	line: '\'\'\'Lots of string\'\'\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.python' }
+	]}],
+
+	[{
+	line: '"""Lots \'\'\'     \'\'\'"""',
+	tokens: [
+		{ startIndex: 0, type: 'string.python' }
+	]}],
+
+	[{
+	line: '\'\'\'Lots \'\'\'0.3e-5',
+	tokens: [
+		{ startIndex: 0, type: 'string.python' },
+		{ startIndex: 11, type: 'number.python' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0xAcBFd',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.python' }
+	]}],
+
+	[{
+	line: '0x0cH',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.python' },
+		{ startIndex: 4, type: 'identifier.python' }
+	]}],
+
+	[{
+	line: '456.7e-7j',
+	tokens: [
+		{ startIndex: 0, type: 'number.python' }
+	]}]
+]);

test/r.test.ts

@@ -0,0 +1,471 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('r', [
+	// Keywords
+	[{
+	line: 'function(a) { a }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.r' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 9, type: 'identifier.r' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 11, type: 'white.r' },
+		{ startIndex: 12, type: 'delimiter.curly.r' },
+		{ startIndex: 13, type: 'white.r' },
+		{ startIndex: 14, type: 'identifier.r' },
+		{ startIndex: 15, type: 'white.r' },
+		{ startIndex: 16, type: 'delimiter.curly.r' }
+	]}],
+
+	[{
+	line: 'while(FALSE) { break }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.r' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 6, type: 'constant.r' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 12, type: 'white.r' },
+		{ startIndex: 13, type: 'delimiter.curly.r' },
+		{ startIndex: 14, type: 'white.r' },
+		{ startIndex: 15, type: 'keyword.r' },
+		{ startIndex: 20, type: 'white.r' },
+		{ startIndex: 21, type: 'delimiter.curly.r' }
+	]}],
+
+	[{
+	line: 'if (a) { b } else { d }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.r' },
+		{ startIndex: 2, type: 'white.r' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 4, type: 'identifier.r' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 6, type: 'white.r' },
+		{ startIndex: 7, type: 'delimiter.curly.r' },
+		{ startIndex: 8, type: 'white.r' },
+		{ startIndex: 9, type: 'identifier.r' },
+		{ startIndex: 10, type: 'white.r' },
+		{ startIndex: 11, type: 'delimiter.curly.r' },
+		{ startIndex: 12, type: 'white.r' },
+		{ startIndex: 13, type: 'keyword.r' },
+		{ startIndex: 17, type: 'white.r' },
+		{ startIndex: 18, type: 'delimiter.curly.r' },
+		{ startIndex: 19, type: 'white.r' },
+		{ startIndex: 20, type: 'identifier.r' },
+		{ startIndex: 21, type: 'white.r' },
+		{ startIndex: 22, type: 'delimiter.curly.r' }
+	]}],
+
+	// Identifiers
+	[{
+	line: 'a',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' }
+	]}],
+
+	// Comments
+	[{
+	line: ' # comment #',
+	tokens: [
+		{ startIndex: 0, type: 'white.r' },
+		{ startIndex: 1, type: 'comment.r' }
+	]}],
+
+	// Roxygen comments
+	[{
+	line: ' #\' @author: me ',
+	tokens: [
+		{ startIndex: 0, type: 'white.r' },
+		{ startIndex: 1, type: 'comment.doc.r' },
+		{ startIndex: 4, type: 'tag.r' },
+		{ startIndex: 11, type: 'comment.doc.r' }
+	]}],
+
+	// Strings
+	[{
+	line: '"a\\n"',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.r' },
+		{ startIndex: 1, type: 'string.r' },
+		{ startIndex: 4, type: 'string.escape.r' }
+	]}],
+
+	// '\\s' is not a special character
+	[{
+	line: '"a\\s"',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.r' },
+		{ startIndex: 1, type: 'string.r' },
+		{ startIndex: 2, type: 'error-token.r' },
+		{ startIndex: 4, type: 'string.escape.r' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1e10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1e-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1e10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1e-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1E10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1E-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1E10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1E-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	// Operators
+	[{
+	line: 'a & b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a - b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a * b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a + b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a = b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a | b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ! b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a < b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a > b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ^ b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ~ b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a / b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a : b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a %in% b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 6, type: 'white.r' },
+		{ startIndex: 7, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a %->% b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 6, type: 'white.r' },
+		{ startIndex: 7, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a == b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a != b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a %% b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a && b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a || b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a <- b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a <<- b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 5, type: 'white.r' },
+		{ startIndex: 6, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a -> b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ->> b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 5, type: 'white.r' },
+		{ startIndex: 6, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a $ b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a << b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a >> b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}]
+]);

test/ruby.test.ts

@@ -0,0 +1,125 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('ruby', [
+	// Keywords
+	[{
+	line: 'class Klass def init() end',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.class.ruby' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'constructor.identifier.ruby' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.def.ruby' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.ruby' },
+		{ startIndex: 20, type: 'delimiter.parenthesis.ruby' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'keyword.def.ruby' }
+	]}],
+
+	// Single digit
+	[{
+	line: 'x == 1 ',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'operator.ruby' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.ruby' },
+		{ startIndex: 6, type: '' }
+	]}],
+
+	// Regex
+	[{
+	line: 'text =~ /Ruby/',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'operator.ruby' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'regexp.delim.ruby' },
+		{ startIndex: 9, type: 'regexp.ruby' },
+		{ startIndex: 13, type: 'regexp.delim.ruby' }
+	]}],
+
+	[{
+	line: 'text.sub!(/Rbuy/, "Ruby")',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.ruby' },
+		{ startIndex: 9, type: 'delimiter.parenthesis.ruby' },
+		{ startIndex: 10, type: 'regexp.delim.ruby' },
+		{ startIndex: 11, type: 'regexp.ruby' },
+		{ startIndex: 15, type: 'regexp.delim.ruby' },
+		{ startIndex: 16, type: 'delimiter.ruby' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.d.delim.ruby' },
+		{ startIndex: 19, type: 'string.$S2.ruby' },
+		{ startIndex: 23, type: 'string.d.delim.ruby' },
+		{ startIndex: 24, type: 'delimiter.parenthesis.ruby' }
+	]}],
+
+	// make sure that division does not match regex
+	[{
+	line: 'a / b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'operator.ruby' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.ruby' }
+	]}],
+
+	// Heredoc
+	[{
+	line: '<<HERE',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.delimiter.ruby' }
+	]}, {
+	line: 'do some string',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.ruby' }
+	]}, {
+	line: 'HERE',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.delimiter.ruby' }
+	]}],
+
+	[{
+	line: 'x <<HERE',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: 'string.heredoc.delimiter.ruby' }
+	]}, {
+	line: 'do some string',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.ruby' }
+	]}, {
+	line: 'HERE',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.delimiter.ruby' }
+	]}],
+
+	[{
+	line: 'x<<HERE',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: 'operator.ruby' },
+		{ startIndex: 3, type: 'constructor.identifier.ruby' }
+	]}],
+
+	[{
+	line: 'x<<-HERE',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: 'string.heredoc.delimiter.ruby' }
+	]}]
+]);

test/sql.test.ts

@@ -0,0 +1,587 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('sql', [
+	// Comments
+	[{
+	line: '-- a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.sql' }
+	]}],
+
+	[{
+	line: '---sticky -- comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.sql' }
+	]}],
+
+	[{
+	line: '-almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'operator.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.sql' },
+		{ startIndex: 9, type: 'white.sql' },
+		{ startIndex: 10, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '/* a full line comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.quote.sql' },
+		{ startIndex: 2, type: 'comment.sql' },
+		{ startIndex: 23, type: 'comment.quote.sql' }
+	]}],
+
+	[{
+	line: '/* /// *** /// */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.quote.sql' },
+		{ startIndex: 2, type: 'comment.sql' },
+		{ startIndex: 15, type: 'comment.quote.sql' }
+	]}],
+
+	[{
+	line: 'declare @x int = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.sql' },
+		{ startIndex: 10, type: 'white.sql' },
+		{ startIndex: 11, type: 'keyword.sql' },
+		{ startIndex: 14, type: 'white.sql' },
+		{ startIndex: 15, type: 'operator.sql' },
+		{ startIndex: 16, type: 'white.sql' },
+		{ startIndex: 17, type: 'comment.quote.sql' },
+		{ startIndex: 19, type: 'comment.sql' },
+		{ startIndex: 37, type: 'comment.quote.sql' },
+		{ startIndex: 39, type: 'white.sql' },
+		{ startIndex: 40, type: 'number.sql' },
+		{ startIndex: 41, type: 'delimiter.sql' }
+	]}],
+
+	// Not supporting nested comments, as nested comments seem to not be standard?
+	// i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic
+	[{
+	line: '@x=/* a /* nested comment  1*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 3, type: 'comment.quote.sql' },
+		{ startIndex: 5, type: 'comment.sql' },
+		{ startIndex: 28, type: 'comment.quote.sql' },
+		{ startIndex: 30, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '@x=/* another comment */ 1*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 3, type: 'comment.quote.sql' },
+		{ startIndex: 5, type: 'comment.sql' },
+		{ startIndex: 22, type: 'comment.quote.sql' },
+		{ startIndex: 24, type: 'white.sql' },
+		{ startIndex: 25, type: 'number.sql' },
+		{ startIndex: 26, type: 'operator.sql' },
+		{ startIndex: 28, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '@x=/*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 3, type: 'comment.quote.sql' },
+		{ startIndex: 5, type: 'comment.sql' }
+	]}],
+
+	// Numbers
+	[{
+	line: '123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '-123',
+	tokens: [
+		{ startIndex: 0, type: 'operator.sql' },
+		{ startIndex: 1, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0xaBc123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0XaBc123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0x',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0x0',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0xAB_CD',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' },
+		{ startIndex: 4, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '$',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$-123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$-+-123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$123.5678',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$0.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$99.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$0.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$.0',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '123.5678',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '99.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.0',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1E-2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1E+2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0.1E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1.E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.1E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	// Identifiers
+	[{
+	line: '_abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '#abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '##abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '@abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '@@abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '$abc',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '$action',
+	tokens: [
+		{ startIndex: 0, type: 'predefined.sql' }
+	]}],
+
+	[{
+	line: '$nonexistent',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '@@DBTS',
+	tokens: [
+		{ startIndex: 0, type: 'predefined.sql' }
+	]}],
+
+	[{
+	line: '@@nonexistent',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'declare [abc 321];',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.quote.sql' },
+		{ startIndex: 9, type: 'identifier.sql' },
+		{ startIndex: 16, type: 'identifier.quote.sql' },
+		{ startIndex: 17, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '[abc[[ 321 ]] xyz]',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 17, type: 'identifier.quote.sql' }
+	]}],
+
+	[{
+	line: '[abc',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'declare "abc 321";',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.quote.sql' },
+		{ startIndex: 9, type: 'identifier.sql' },
+		{ startIndex: 16, type: 'identifier.quote.sql' },
+		{ startIndex: 17, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '"abc"" 321 "" xyz"',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 17, type: 'identifier.quote.sql' }
+	]}],
+
+	[{
+	line: '"abc',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'int',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' }
+	]}],
+
+	[{
+	line: '[int]',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 4, type: 'identifier.quote.sql' }
+	]}],
+
+	// Strings
+	[{
+	line: 'declare @x=\'a string\';',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.sql' },
+		{ startIndex: 10, type: 'operator.sql' },
+		{ startIndex: 11, type: 'string.quote.sql' },
+		{ startIndex: 12, type: 'string.sql' },
+		{ startIndex: 20, type: 'string.quote.sql' },
+		{ startIndex: 21, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '\'a \'\' string with quotes\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' },
+		{ startIndex: 24, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: '\'a " string with quotes\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' },
+		{ startIndex: 23, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: '\'a -- string with comment\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' },
+		{ startIndex: 25, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: 'N\'a unicode string\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 2, type: 'string.sql' },
+		{ startIndex: 18, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: '\'a endless string',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' }
+	]}],
+
+	// Operators
+	[{
+	line: 'SET @x=@x+1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 3, type: 'white.sql' },
+		{ startIndex: 4, type: 'identifier.sql' },
+		{ startIndex: 6, type: 'operator.sql' },
+		{ startIndex: 7, type: 'identifier.sql' },
+		{ startIndex: 9, type: 'operator.sql' },
+		{ startIndex: 10, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '@x^=@x',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 4, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'WHERE x IS NOT NULL',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 5, type: 'white.sql' },
+		{ startIndex: 6, type: 'identifier.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'operator.sql' },
+		{ startIndex: 10, type: 'white.sql' },
+		{ startIndex: 11, type: 'operator.sql' },
+		{ startIndex: 14, type: 'white.sql' },
+		{ startIndex: 15, type: 'operator.sql' }
+	]}],
+
+	[{
+	line: 'SELECT * FROM dbo.MyTable WHERE MyColumn IN (1,2)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 6, type: 'white.sql' },
+		{ startIndex: 7, type: 'operator.sql' },
+		{ startIndex: 8, type: 'white.sql' },
+		{ startIndex: 9, type: 'keyword.sql' },
+		{ startIndex: 13, type: 'white.sql' },
+		{ startIndex: 14, type: 'identifier.sql' },
+		{ startIndex: 17, type: 'delimiter.sql' },
+		{ startIndex: 18, type: 'identifier.sql' },
+		{ startIndex: 25, type: 'white.sql' },
+		{ startIndex: 26, type: 'keyword.sql' },
+		{ startIndex: 31, type: 'white.sql' },
+		{ startIndex: 32, type: 'identifier.sql' },
+		{ startIndex: 40, type: 'white.sql' },
+		{ startIndex: 41, type: 'operator.sql' },
+		{ startIndex: 43, type: 'white.sql' },
+		{ startIndex: 44, type: 'delimiter.parenthesis.sql' },
+		{ startIndex: 45, type: 'number.sql' },
+		{ startIndex: 46, type: 'delimiter.sql' },
+		{ startIndex: 47, type: 'number.sql' },
+		{ startIndex: 48, type: 'delimiter.parenthesis.sql' }
+	]}],
+
+	// Scopes
+	[{
+	line: 'WHILE() BEGIN END',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'keyword.block.sql' },
+		{ startIndex: 13, type: 'white.sql' },
+		{ startIndex: 14, type: 'keyword.block.sql' }
+	]}],
+
+	[{
+	line: 'BEGIN TRAN BEGIN TRY SELECT $ COMMIT END TRY BEGIN CATCH ROLLBACK END CATCH',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 10, type: 'white.sql' },
+		{ startIndex: 11, type: 'keyword.try.sql' },
+		{ startIndex: 20, type: 'white.sql' },
+		{ startIndex: 21, type: 'keyword.sql' },
+		{ startIndex: 27, type: 'white.sql' },
+		{ startIndex: 28, type: 'number.sql' },
+		{ startIndex: 29, type: 'white.sql' },
+		{ startIndex: 30, type: 'keyword.sql' },
+		{ startIndex: 36, type: 'white.sql' },
+		{ startIndex: 37, type: 'keyword.try.sql' },
+		{ startIndex: 44, type: 'white.sql' },
+		{ startIndex: 45, type: 'keyword.catch.sql' },
+		{ startIndex: 56, type: 'white.sql' },
+		{ startIndex: 57, type: 'keyword.sql' },
+		{ startIndex: 65, type: 'white.sql' },
+		{ startIndex: 66, type: 'keyword.catch.sql' }
+	]}],
+
+	[{
+	line: 'SELECT CASE $ WHEN 3 THEN 4 ELSE 5 END',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 6, type: 'white.sql' },
+		{ startIndex: 7, type: 'keyword.block.sql' },
+		{ startIndex: 11, type: 'white.sql' },
+		{ startIndex: 12, type: 'number.sql' },
+		{ startIndex: 13, type: 'white.sql' },
+		{ startIndex: 14, type: 'keyword.choice.sql' },
+		{ startIndex: 18, type: 'white.sql' },
+		{ startIndex: 19, type: 'number.sql' },
+		{ startIndex: 20, type: 'white.sql' },
+		{ startIndex: 21, type: 'keyword.choice.sql' },
+		{ startIndex: 25, type: 'white.sql' },
+		{ startIndex: 26, type: 'number.sql' },
+		{ startIndex: 27, type: 'white.sql' },
+		{ startIndex: 28, type: 'keyword.sql' },
+		{ startIndex: 32, type: 'white.sql' },
+		{ startIndex: 33, type: 'number.sql' },
+		{ startIndex: 34, type: 'white.sql' },
+		{ startIndex: 35, type: 'keyword.block.sql' }
+	]}]
+]);

test/swift.test.ts

@@ -0,0 +1,181 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('swift', [
+
+	// Attributes
+	[{
+	line: '@noescape',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.control.swift' } /* '@noescape' */
+	]}],
+	//Keyword and Type Identifier
+	[{
+		line: 'class App: UI, UIApp, UIView {',
+		tokens: [
+		{ startIndex: 0, type: 'keyword.swift' } /* 'class' */,
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'type.identifier.swift' } /* 'App' */,
+		{ startIndex: 9, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'type.identifier.swift' } /* 'UI' */,
+		{ startIndex: 13, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'type.identifier.swift' } /* 'UIApp' */,
+		{ startIndex: 20, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'type.identifier.swift' } /* 'UIView' */,
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'delimiter.curly.swift' } /* '{' */
+	]}],
+	// Keyword, Identifier, and Type Identifier
+	[{
+		line: '    var window: UIWindow?',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.swift' } /* 'var' */,
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.swift' } /* 'window' */,
+		{ startIndex: 14, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'type.identifier.swift' } /* 'UIWindow' */,
+		{ startIndex: 24, type: 'keyword.operator.swift' } /* '?' */
+	]}],
+	//Comment
+	[{
+		line: '    // Comment',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.swift' } /* '// Comment' */
+	]}],
+	//Block Comment with Embedded Comment followed by code
+	[{
+		line: '    /* Comment //Embedded */ var y = 0b10',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.swift' }, // /* '/* Comment //Embedded */' */,
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'keyword.swift' } /* 'var' */,
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'identifier.swift' }	/* 'y' */,
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'number.binary.swift' } /* '0b10' */
+	]}],
+	// Method signature (broken on two lines)
+	[{
+		line: '    public func app(app: App, opts:',
+		tokens:[
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.swift' } /* 'public' */,
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.swift' } /* 'func' */,
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.swift' } /* 'app' */,
+		{ startIndex: 19, type: 'delimiter.parenthesis.swift' } /* '(' */,
+		{ startIndex: 20, type: 'identifier.swift' }/* 'app' */,
+		{ startIndex: 23, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'type.identifier.swift' } /* 'App' */,
+		{ startIndex: 28, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'identifier.swift' } /* 'opts' */,
+		{ startIndex: 34, type: 'keyword.operator.swift' } /* ':' */,
+	]}],
+	// Method signature Continued
+	[{
+		line: '        [NSObject: AnyObject]?) -> Bool {',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.square.swift' } /* '[' */,
+		{ startIndex: 9, type: 'type.identifier.swift' } /* 'NSObject' */,
+		{ startIndex: 17, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'type.identifier.swift' } /* 'AnyObject' */,
+		{ startIndex: 28, type: 'delimiter.square.swift' } /* ']' */,
+		{ startIndex: 29, type: 'keyword.operator.swift' } /* '?' */,
+		{ startIndex: 30, type: 'delimiter.parenthesis.swift' } /* ')' */,
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'keyword.operator.swift' } /* '->' */,
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'type.identifier.swift' } /* 'Bool' */,
+		{ startIndex: 39, type: '' },
+		{ startIndex: 40, type: 'delimiter.curly.swift' } /* '{' */
+	]}],
+	// String with escapes
+	[{
+		line: '        var `String` = "String w/ \\"escape\\""',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.swift' } /* 'var' */,
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.operator.swift' } /* '`' */,
+		{ startIndex: 13, type: 'identifier.swift' } /* 'String' */,
+		{ startIndex: 19, type: 'keyword.operator.swift' } /* '`' */,
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'string.quote.swift' } /* '"' */,
+		{ startIndex: 24, type: 'string.swift' } /* 'String w/ \\"escape\\""' */,
+		{ startIndex: 44, type: 'string.quote.swift' } /* '"' */,
+	]}],
+	// String with interpolated expression
+	[{
+		line: '        let message = "\\(y) times 2.5 is \\(Double(25) * 2.5)"',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.swift' } /* 'let' */,
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.swift' } /* 'message' */,
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'string.quote.swift' } /* '"' */,
+		{ startIndex: 23, type: 'keyword.operator.swift' } /* '\(' */,
+		{ startIndex: 25, type: 'identifier.swift' },
+		{ startIndex: 26, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 27, type: 'string.swift' } /* ' times 2.5 is ' */,
+		{ startIndex: 41, type: 'keyword.operator.swift' } /* '\(' */,
+		{ startIndex: 43, type: 'type.identifier.swift' } /* 'Double' */,
+		{ startIndex: 49, type: 'keyword.operator.swift' } /* '(' */,
+		{ startIndex: 50, type: 'number.swift' } /* '25' */,
+		{ startIndex: 52, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 53, type: '' },
+		{ startIndex: 54, type: 'keyword.operator.swift' } /* '*' */,
+		{ startIndex: 55, type: '' },
+		{ startIndex: 56, type: 'number.float.swift' } /* '2.5' */,
+		{ startIndex: 59, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 60, type: 'string.quote.swift' } /* '"' */
+	]}],
+	// Method invocation/property accessor.
+	[{
+		line: '        let view = self.window!.contr as! UIView',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.swift' } /* 'let' */,
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.swift' } /* 'view' */,
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.swift' } /* 'self' */,
+		{ startIndex: 23, type: 'delimeter.swift' } /* '.' */,
+		{ startIndex: 24, type: 'type.identifier.swift' } /* 'window' */,
+		{ startIndex: 30, type: 'keyword.operator.swift' } /* '!' */,
+		{ startIndex: 31, type: 'delimeter.swift' } /* '.' */,
+		{ startIndex: 32, type: 'type.identifier.swift' } /* 'contr' */,
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'keyword.swift' } /* 'as' */,
+		{ startIndex: 40, type: 'keyword.operator.swift' } /* '!' */,
+		{ startIndex: 41, type: '' },
+		{ startIndex: 42, type: 'type.identifier.swift' } /* 'UIView' */
+	]}]
+]);
+

test/testRunner.ts

@@ -0,0 +1,58 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import {loadLanguage} from '../src/monaco.contribution';
+import * as assert from 'assert';
+
+// Allow for running under nodejs/requirejs in tests
+var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+export interface IRelaxedToken {
+	startIndex: number;
+	type: string;
+}
+
+export interface ITestItem {
+	line: string;
+	tokens: IRelaxedToken[];
+}
+
+export function testTokenization(languageId:string, tests:ITestItem[][]): void {
+	suite(languageId + ' tokenization', () => {
+		test('', (done) => {
+			loadLanguage(languageId).then(() => {
+				runTests(languageId, tests);
+				done();
+			}).then(null, done);
+		});
+	});
+}
+
+function runTests(languageId:string, tests:ITestItem[][]): void {
+	tests.forEach((test) => runTest(languageId, test));
+}
+
+function runTest(languageId:string, test:ITestItem[]): void {
+	let text = test.map(t => t.line).join('\n');
+	let model = _monaco.editor.createModel(text, languageId);
+
+	for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber <= lineCount; lineNumber++) {
+		let actual: IRelaxedToken[] = [];
+		let lineTokens = (<any>model).getLineTokens(lineNumber);
+		for (let j = 0; j < lineTokens.getTokenCount(); j++) {
+			actual.push({
+				startIndex: lineTokens.getTokenStartIndex(j),
+				type: lineTokens.getTokenType(j)
+			});
+		}
+
+		let expected = test[lineNumber - 1].tokens;
+		// console.log(`actual: ${JSON.stringify(actual)}`);
+		// console.log(`expected: ${JSON.stringify(expected)}`);
+		assert.deepEqual(actual, expected, 'TOKENIZING ' + text);
+	}
+
+	model.dispose();
+}

test/vb.test.ts

@@ -0,0 +1,395 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('vb', [
+
+	// Comments - single line
+	[{
+	line: '\'',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '    \' a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '\' a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '\'sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '1 \' 2; \' comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: 'Dim x = 1; \' my comment \'\' is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dim.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.vb' },
+		{ startIndex: 9, type: 'delimiter.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: 'REM this is a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '2 + 5 REM comment starts',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'comment.vb' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' }
+	]}],
+
+	[{
+	line: '0.0',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '&h123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.vb' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5R',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3#',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3!',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: 'delimiter.vb' },
+		{ startIndex: 2, type: 'number.vb' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 3, type: 'delimiter.vb' },
+		{ startIndex: 4, type: 'number.vb' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.vb' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'Imports Microsoft.VisualBasic',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.imports.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.vb' },
+		{ startIndex: 17, type: 'delimiter.vb' },
+		{ startIndex: 18, type: 'identifier.vb' }
+	]}],
+
+	[{
+	line: 'Private Sub Foo(ByVal sender As String)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.private.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.tag-sub.vb' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.vb' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.vb' },
+		{ startIndex: 16, type: 'keyword.byval.vb' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'identifier.vb' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'keyword.as.vb' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'keyword.string.vb' },
+		{ startIndex: 38, type: 'delimiter.parenthesis.vb' }
+	]}],
+
+	// Strings
+	[{
+	line: 'String s = "string"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.string.vb' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.vb' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.vb' }
+	]}],
+
+	[{
+	line: '"use strict";',
+	tokens: [
+		{ startIndex: 0, type: 'string.vb' },
+		{ startIndex: 12, type: 'delimiter.vb' }
+	]}],
+
+	// Tags
+	[{
+	line: 'Public Sub ToString()',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.public.vb' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.tag-sub.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'identifier.vb' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.vb' }
+	]}],
+
+	[{
+	line: 'public sub ToString()',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.public.vb' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.tag-sub.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'identifier.vb' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.vb' }
+	]}],
+
+	[{
+	line: 'While Do Continue While End While',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-while.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'keyword.tag-do.vb' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.tag-continue.vb' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'keyword.tag-while.vb' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'keyword.tag-while.vb' }
+	]}],
+
+	[{
+	line: 'While while WHILE WHile whiLe',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-while.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'keyword.tag-while.vb' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.tag-while.vb' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'keyword.tag-while.vb' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'keyword.tag-while.vb' }
+	]}],
+
+	[{
+	line: 'If b(i) = col Then',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-if.vb' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'identifier.vb' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.vb' },
+		{ startIndex: 5, type: 'identifier.vb' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.vb' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.then.vb' }
+	]}],
+
+	[{
+	line: 'Do stuff While True Loop',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-do.vb' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'identifier.vb' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.tag-while.vb' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.true.vb' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'keyword.tag-do.vb' }
+	]}],
+
+	[{
+	line: 'For i = 0 To 10 DoStuff Next',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-for.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.to.vb' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'number.vb' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.vb' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'keyword.tag-for.vb' }
+	]}],
+
+	[{
+	line: 'For stuff End For',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-for.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.end.vb' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.tag-for.vb' }
+	]}],
+
+	[{
+	line: 'For stuff end for',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-for.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.end.vb' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.tag-for.vb' }
+	]}]
+]);

test/xml.test.ts

@@ -0,0 +1,573 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('xml', [
+	// Complete Start Tag with Whitespace
+	[{
+	line: '<person>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 7, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<person/>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 8, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<person >',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<person />',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'tag.tag-person.xml' },
+		{ startIndex: 9, type: 'delimiter.start.xml' }
+	]}],
+
+	// Incomplete Start Tag
+	[{
+	line: '<',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '<person',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' }
+	]}],
+
+	[{
+	line: '<input',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-input.xml' }
+	]}],
+
+	// Invalid Open Start Tag
+	[{
+	line: '< person',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '< person>',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: 'i <person;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'tag.tag-person.xml' },
+		{ startIndex: 9, type: '' }
+	]}],
+
+	// Tag with Attribute
+	[{
+	line: '<tool name="">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 13, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name="Monaco">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name=\'Monaco\'>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Attributes
+	[{
+	line: '<tool name="Monaco" version="1.0">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'attribute.name.xml' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'attribute.value.xml' },
+		{ startIndex: 33, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Name-Only-Attribute
+	[{
+	line: '<tool name>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name version>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.name.xml' },
+		{ startIndex: 18, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Attribute And Whitespace
+	[{
+	line: '<tool name=  "monaco">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 13, type: 'attribute.value.xml' },
+		{ startIndex: 21, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name = "monaco">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 13, type: 'attribute.value.xml' },
+		{ startIndex: 21, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Invalid Attribute Name
+	[{
+	line: '<tool name!@#="bar">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 15, type: 'attribute.name.xml' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Invalid Attribute Value
+	[{
+	line: '<tool name=">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 12, type: 'delimiter.start.xml' }
+	]}],
+
+	// Complete End Tag
+	[{
+	line: '</person>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-person.xml' },
+		{ startIndex: 8, type: 'delimiter.end.xml' }
+	]}],
+
+	// Complete End Tag with Whitespace
+	[{
+	line: '</person  >',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-person.xml' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 10, type: 'delimiter.end.xml' }
+	]}],
+
+	// Incomplete End Tag
+	[{
+	line: '</person',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	// Comments
+	[{
+	line: '<!-- -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.xml' },
+		{ startIndex: 4, type: 'comment.content.xml' },
+		{ startIndex: 5, type: 'comment.xml' }
+	]}],
+
+	[{
+	line: '<!--a>monaco</a -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.xml' },
+		{ startIndex: 4, type: 'comment.content.xml' },
+		{ startIndex: 16, type: 'comment.xml' }
+	]}],
+
+	[{
+	line: '<!--a>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.xml' },
+		{ startIndex: 4, type: 'comment.content.xml' }
+	]},{
+	line: 'monaco ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' }
+	]},{
+	line: 'tools</a -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' },
+		{ startIndex: 9, type: 'comment.xml' }
+	]}],
+
+	// CDATA
+	[{
+	line: '<tools><![CDATA[<person/>]]></tools>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tools.xml' },
+		{ startIndex: 6, type: 'delimiter.start.xml' },
+		{ startIndex: 7, type: 'delimiter.cdata.xml' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 25, type: 'delimiter.cdata.xml' },
+		{ startIndex: 28, type: 'delimiter.end.xml' },
+		{ startIndex: 30, type: 'tag.tag-tools.xml' },
+		{ startIndex: 35, type: 'delimiter.end.xml' }
+	]}],
+
+	[{
+	line: '<tools>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tools.xml' },
+		{ startIndex: 6, type: 'delimiter.start.xml' }
+	]},{
+	line: '\t<![CDATA[',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.cdata.xml' }
+	]},{
+	line: '\t\t<person/>',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]},{
+	line: '\t]]>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.cdata.xml' },
+	]},{
+	line: '</tools>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-tools.xml' },
+		{ startIndex: 7, type: 'delimiter.end.xml' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: '<?xml version="1.0"?>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'metatag.instruction.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}, {
+	line: '<configuration xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-configuration.xml' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'attribute.name.xml' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'attribute.value.xml' },
+		{ startIndex: 78, type: 'delimiter.start.xml' }
+	]}, {
+	line: '  <connectionStrings>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'tag.tag-connectionstrings.xml' },
+		{ startIndex: 20, type: 'delimiter.start.xml' }
+	]}, {
+	line: '    <add name="MyDB" ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.start.xml' },
+		{ startIndex: 5, type: 'tag.tag-add.xml' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'attribute.name.xml' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'attribute.value.xml' },
+		{ startIndex: 20, type: '' }
+	]}, {
+	line: '      connectionString="value for the deployed Web.config file" ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'attribute.value.xml' },
+		{ startIndex: 63, type: '' }
+	]}, {
+	line: '      xdt:Transform="SetAttributes" xdt:Locator="Match(name)"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'attribute.value.xml' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 36, type: 'attribute.name.xml' },
+		{ startIndex: 47, type: '' },
+		{ startIndex: 48, type: 'attribute.value.xml' },
+		{ startIndex: 61, type: 'tag.tag-add.xml' },
+		{ startIndex: 62, type: 'delimiter.start.xml' }
+	]}, {
+	line: '  </connectionStrings>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.end.xml' },
+		{ startIndex: 4, type: 'tag.tag-connectionstrings.xml' },
+		{ startIndex: 21, type: 'delimiter.end.xml' }
+	]}, {
+	line: '  <system.web>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'tag.tag-system.web.xml' },
+		{ startIndex: 13, type: 'delimiter.start.xml' }
+	]}, {
+	line: '    <customErrors defaultRedirect="GenericError.htm"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.start.xml' },
+		{ startIndex: 5, type: 'tag.tag-customerrors.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'attribute.name.xml' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 34, type: 'attribute.value.xml' }
+	]}, {
+	line: '      mode="RemoteOnly" xdt:Transform="Replace">',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.name.xml' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'attribute.value.xml' },
+		{ startIndex: 47, type: 'delimiter.start.xml' }
+	]}, {
+	line: '      <error statusCode="500" redirect="InternalError.htm"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.start.xml' },
+		{ startIndex: 7, type: 'tag.tag-error.xml' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'attribute.name.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.value.xml' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'attribute.name.xml' },
+		{ startIndex: 38, type: '' },
+		{ startIndex: 39, type: 'attribute.value.xml' },
+		{ startIndex: 58, type: 'tag.tag-error.xml' },
+		{ startIndex: 59, type: 'delimiter.start.xml' }
+	]}, {
+	line: '    </customErrors>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.end.xml' },
+		{ startIndex: 6, type: 'tag.tag-customerrors.xml' },
+		{ startIndex: 18, type: 'delimiter.end.xml' }
+	]}, {
+	line: '  </system.web>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.end.xml' },
+		{ startIndex: 4, type: 'tag.tag-system.web.xml' },
+		{ startIndex: 14, type: 'delimiter.end.xml' }
+	]}, {
+	line: '	',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '	<!-- The stuff below was added for extra tokenizer testing -->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'comment.xml' },
+		{ startIndex: 5, type: 'comment.content.xml' },
+		{ startIndex: 60, type: 'comment.xml' }
+	]}, {
+	line: '	<!-- A multi-line comment <with> </with>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'comment.xml' },
+		{ startIndex: 5, type: 'comment.content.xml' }
+	]}, {
+	line: '       <tags>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' }
+	]}, {
+	line: '				 -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' },
+		{ startIndex: 5, type: 'comment.xml' }
+	]}, {
+	line: '	<!DOCTYPE another meta tag>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'metatag.declaration.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.name.xml' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'attribute.name.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.name.xml' },
+		{ startIndex: 27, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<tools><![CDATA[Some text and tags <person/>]]></tools>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-tools.xml' },
+		{ startIndex: 7, type: 'delimiter.start.xml' },
+		{ startIndex: 8, type: 'delimiter.cdata.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 45, type: 'delimiter.cdata.xml' },
+		{ startIndex: 48, type: 'delimiter.end.xml' },
+		{ startIndex: 50, type: 'tag.tag-tools.xml' },
+		{ startIndex: 55, type: 'delimiter.end.xml' }
+	]}, {
+	line: '	<aSelfClosingTag with="attribute" />',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'attribute.name.xml' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'attribute.value.xml' },
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 36, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<aSelfClosingTag with="attribute"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'attribute.name.xml' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'attribute.value.xml' },
+		{ startIndex: 34, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 35, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<namespace:aSelfClosingTag otherspace:with="attribute"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-namespace:aselfclosingtag.xml' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'attribute.name.xml' },
+		{ startIndex: 43, type: '' },
+		{ startIndex: 44, type: 'attribute.value.xml' },
+		{ startIndex: 55, type: 'tag.tag-namespace:aselfclosingtag.xml' },
+		{ startIndex: 56, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<valid-name also_valid this.one=\'too is valid\'/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-valid-name.xml' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'attribute.name.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.name.xml' },
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'attribute.value.xml' },
+		{ startIndex: 47, type: 'tag.tag-valid-name.xml' },
+		{ startIndex: 48, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<aSimpleSelfClosingTag />',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-asimpleselfclosingtag.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'tag.tag-asimpleselfclosingtag.xml' },
+		{ startIndex: 25, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<aSimpleSelfClosingTag/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-asimpleselfclosingtag.xml' },
+		{ startIndex: 24, type: 'delimiter.start.xml' }
+	]}, {
+	line: '</configuration>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-configuration.xml' },
+		{ startIndex: 15, type: 'delimiter.end.xml' }
+	]}]
+]);

tsconfig.json

@@ -0,0 +1,58 @@
+{
+  "compilerOptions": {
+    "module": "amd",
+    "outDir": "out",
+    "target": "es5"
+  },
+  "filesGlob": [
+    "src/*.ts",
+    "test/*.ts",
+    "node_modules/monaco-editor-core/monaco.d.ts"
+  ],
+  "files": [
+    "src/bat.ts",
+    "src/coffee.ts",
+    "src/cpp.ts",
+    "src/csharp.ts",
+    "src/dockerfile.ts",
+    "src/fsharp.ts",
+    "src/go.ts",
+    "src/ini.ts",
+    "src/jade.ts",
+    "src/java.ts",
+    "src/lua.ts",
+    "src/monaco.contribution.ts",
+    "src/objective-c.ts",
+    "src/powershell.ts",
+    "src/python.ts",
+    "src/r.ts",
+    "src/ruby.ts",
+    "src/sql.ts",
+    "src/swift.ts",
+    "src/vb.ts",
+    "src/xml.ts",
+    "test/assert.d.ts",
+    "test/bat.test.ts",
+    "test/coffee.test.ts",
+    "test/cpp.test.ts",
+    "test/csharp.test.ts",
+    "test/dockerfile.test.ts",
+    "test/fsharp.test.ts",
+    "test/go.test.ts",
+    "test/jade.test.ts",
+    "test/java.test.ts",
+    "test/lua.test.ts",
+    "test/mocha.d.ts",
+    "test/objective-c.test.ts",
+    "test/powershell.test.ts",
+    "test/python.test.ts",
+    "test/r.test.ts",
+    "test/ruby.test.ts",
+    "test/sql.test.ts",
+    "test/swift.test.ts",
+    "test/testRunner.ts",
+    "test/vb.test.ts",
+    "test/xml.test.ts",
+    "node_modules/monaco-editor-core/monaco.d.ts"
+  ]
+}