Remove language registration, configuration and tokenization

README.md

@@ -22,7 +22,6 @@ This npm module is bundled and distributed in the [monaco-editor](https://www.np
 * `npm install .`
 * `npm run watch`
 * open `$/monaco-typescript/test/index.html` in your favorite browser.
-* test with `npm run test`
 
 ## Updating TypeScript
 

package-lock.json

@@ -4,164 +4,17 @@
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
-    "balanced-match": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
-      "dev": true
-    },
-    "brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
-      "requires": {
-        "balanced-match": "1.0.0",
-        "concat-map": "0.0.1"
-      }
-    },
-    "browser-stdout": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz",
-      "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==",
-      "dev": true
-    },
-    "commander": {
-      "version": "2.15.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
-      "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==",
-      "dev": true
-    },
-    "concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
-      "dev": true
-    },
-    "debug": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-      "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-      "dev": true,
-      "requires": {
-        "ms": "2.0.0"
-      }
-    },
-    "diff": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
-      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
-      "dev": true
-    },
-    "escape-string-regexp": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
-      "dev": true
-    },
-    "fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
-      "dev": true
-    },
-    "glob": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
-      "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
-      "dev": true,
-      "requires": {
-        "fs.realpath": "1.0.0",
-        "inflight": "1.0.6",
-        "inherits": "2.0.3",
-        "minimatch": "3.0.4",
-        "once": "1.4.0",
-        "path-is-absolute": "1.0.1"
-      }
-    },
-    "growl": {
-      "version": "1.10.5",
-      "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
-      "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==",
-      "dev": true
-    },
-    "has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
-      "dev": true
-    },
-    "he": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz",
-      "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=",
-      "dev": true
-    },
-    "inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
-      "dev": true,
-      "requires": {
-        "once": "1.4.0",
-        "wrappy": "1.0.2"
-      }
-    },
-    "inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
-      "dev": true
-    },
-    "minimatch": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
-      "dev": true,
-      "requires": {
-        "brace-expansion": "1.1.11"
-      }
-    },
-    "minimist": {
-      "version": "0.0.8",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
-      "dev": true
-    },
-    "mkdirp": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
-      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
-      "dev": true,
-      "requires": {
-        "minimist": "0.0.8"
-      }
-    },
-    "mocha": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz",
-      "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==",
-      "dev": true,
-      "requires": {
-        "browser-stdout": "1.3.1",
-        "commander": "2.15.1",
-        "debug": "3.1.0",
-        "diff": "3.5.0",
-        "escape-string-regexp": "1.0.5",
-        "glob": "7.1.2",
-        "growl": "1.10.5",
-        "he": "1.1.1",
-        "minimatch": "3.0.4",
-        "mkdirp": "0.5.1",
-        "supports-color": "5.4.0"
-      }
-    },
     "monaco-editor-core": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/monaco-editor-core/-/monaco-editor-core-0.12.0.tgz",
       "integrity": "sha512-wOoEVAoZtrarDRcQC32Fp0ocacpQd6/Nb0FmUZOHeD3swZuPZhDLOxTyoNLjKq3d+h/6g+IARBLnDaLT5OQD4g==",
       "dev": true
     },
+    "monaco-languages": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/monaco-languages/-/monaco-languages-1.4.0.tgz",
+      "integrity": "sha512-39MZMAEKToktfSpOS0Soj05IvkT5vbebL9AIBGPn8fGi8WgJOcfS3YLiMu07gZKViR7CFZ4RyQJKJHglx0fPdA=="
+    },
     "monaco-plugin-helpers": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/monaco-plugin-helpers/-/monaco-plugin-helpers-1.0.2.tgz",
@@ -179,27 +32,6 @@
         }
       }
     },
-    "ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
-      "dev": true
-    },
-    "once": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
-      "dev": true,
-      "requires": {
-        "wrappy": "1.0.2"
-      }
-    },
-    "path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
-      "dev": true
-    },
     "requirejs": {
       "version": "2.3.5",
       "resolved": "https://registry.npmjs.org/requirejs/-/requirejs-2.3.5.tgz",
@@ -212,15 +44,6 @@
       "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
       "dev": true
     },
-    "supports-color": {
-      "version": "5.4.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-      "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-      "dev": true,
-      "requires": {
-        "has-flag": "3.0.0"
-      }
-    },
     "typescript": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.0.1.tgz",
@@ -244,12 +67,6 @@
           "dev": true
         }
       }
-    },
-    "wrappy": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
-      "dev": true
     }
   }
 }

package.json

@@ -3,7 +3,6 @@
   "version": "3.1.0",
   "description": "TypeScript and JavaScript language support for Monaco Editor",
   "scripts": {
-    "test": "mocha",
     "compile-amd": "mcopy ./src/lib/typescriptServices-amd.js ./release/dev/lib/typescriptServices.js && tsc -p ./src/tsconfig.json",
     "compile-esm": "mcopy ./src/lib/typescriptServices.js ./release/esm/lib/typescriptServices.js && tsc -p ./src/tsconfig.esm.json",
     "compile": "mrmdir ./release && npm run compile-amd && npm run compile-esm",
@@ -21,8 +20,8 @@
     "url": "https://github.com/Microsoft/monaco-typescript/issues"
   },
   "devDependencies": {
-    "mocha": "^5.2.0",
     "monaco-editor-core": "0.12.0",
+    "monaco-languages": "^1.4.0",
     "monaco-plugin-helpers": "^1.0.2",
     "requirejs": "^2.3.5",
     "typescript": "3.0.1",

src/monaco.contribution.ts

@@ -187,24 +187,9 @@ function getMode(): monaco.Promise<typeof mode> {
 	return monaco.Promise.wrap(import('./tsMode'))
 }
 
-monaco.languages.register({
-	id: 'typescript',
-	extensions: ['.ts', '.tsx'],
-	aliases: ['TypeScript', 'ts', 'typescript'],
-	mimetypes: ['text/typescript']
-});
 monaco.languages.onLanguage('typescript', () => {
 	return getMode().then(mode => mode.setupTypeScript(typescriptDefaults));
 });
-
-monaco.languages.register({
-	id: 'javascript',
-	extensions: ['.js', '.es6', '.jsx'],
-	firstLine: '^#!.*\\bnode',
-	filenames: ['jakefile'],
-	aliases: ['JavaScript', 'javascript', 'js'],
-	mimetypes: ['text/javascript'],
-});
 monaco.languages.onLanguage('javascript', () => {
 	return getMode().then(mode => mode.setupJavaScript(javascriptDefaults));
 });

src/test/assert.d.ts

@@ -1,44 +0,0 @@
-declare module "assert" {
-    function internal (value: any, message?: string): void;
-    namespace internal {
-        export class AssertionError implements Error {
-            name: string;
-            message: string;
-            actual: any;
-            expected: any;
-            operator: string;
-            generatedMessage: boolean;
-
-            constructor(options?: {message?: string; actual?: any; expected?: any;
-                                  operator?: string; stackStartFunction?: Function});
-        }
-
-        export function fail(actual?: any, expected?: any, message?: string, operator?: string): void;
-        export function ok(value: any, message?: string): void;
-        export function equal(actual: any, expected: any, message?: string): void;
-        export function notEqual(actual: any, expected: any, message?: string): void;
-        export function deepEqual(actual: any, expected: any, message?: string): void;
-        export function notDeepEqual(acutal: any, expected: any, message?: string): void;
-        export function strictEqual(actual: any, expected: any, message?: string): void;
-        export function notStrictEqual(actual: any, expected: any, message?: string): void;
-        export function deepStrictEqual(actual: any, expected: any, message?: string): void;
-        export function notDeepStrictEqual(actual: any, expected: any, message?: string): void;
-        export var throws: {
-            (block: Function, message?: string): void;
-            (block: Function, error: Function, message?: string): void;
-            (block: Function, error: RegExp, message?: string): void;
-            (block: Function, error: (err: any) => boolean, message?: string): void;
-        };
-
-        export var doesNotThrow: {
-            (block: Function, message?: string): void;
-            (block: Function, error: Function, message?: string): void;
-            (block: Function, error: RegExp, message?: string): void;
-            (block: Function, error: (err: any) => boolean, message?: string): void;
-        };
-
-        export function ifError(value: any): void;
-    }
-
-    export = internal;
-}

src/test/mocha.d.ts

@@ -1,13 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-declare function run(): void;
-
-declare function suite(name: string, fn: (err?)=>void);
-declare function test(name: string, fn: (done?: (err?)=>void)=>void);
-declare function suiteSetup(fn: (done?: (err?)=>void)=>void);
-declare function suiteTeardown(fn: (done?: (err?)=>void)=>void);
-declare function setup(fn: (done?: (err?)=>void)=>void);
-declare function teardown(fn: (done?: (err?)=>void)=>void);

src/test/tokenization.test.ts

@@ -1,502 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import * as assert from 'assert';
-import {createTokenizationSupport, Language} from '../tokenization';
-
-suite('tokenization', () => {
-
-	interface ITestItem {
-		line: string;
-		tokens: monaco.languages.IToken[];
-	}
-
-	function executeTokenizationTests(tests:ITestItem[][]): void {
-		let tokenizationSupport = createTokenizationSupport(Language.EcmaScript5);
-		for (let i = 0, len = tests.length; i < len; i++) {
-			assert.ok(true, 'TEST #' + i);
-			executeTokenizationTest(tokenizationSupport, tests[i]);
-		}
-	}
-
-	function executeTokenizationTest(tokenizationSupport: monaco.languages.TokensProvider, tests:ITestItem[]): void {
-		let state = tokenizationSupport.getInitialState();
-		for (let i = 0, len = tests.length; i < len; i++) {
-			assert.ok(true, tests[i].line);
-
-			let result = tokenizationSupport.tokenize(tests[i].line, state);
-
-			if (tests[i].tokens) {
-				assert.deepEqual(result.tokens, tests[i].tokens, 'Tokenizing line ' + tests[i].line + ': ' + JSON.stringify(tests[i].tokens, null, '\t'));
-			}
-
-			state = result.endState;
-		}
-	}
-
-	test('', () => {
-		executeTokenizationTests([
-
-			// Keywords
-			[{
-			line: 'var x = function() { };',
-			tokens: [
-				{ startIndex: 0, scopes: 'keyword.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'keyword.js' },
-				{ startIndex: 16, scopes: 'delimiter.parenthesis.js' },
-				{ startIndex: 18, scopes: '' },
-				{ startIndex: 19, scopes: 'delimiter.bracket.js' },
-				{ startIndex: 20, scopes: '' },
-				{ startIndex: 21, scopes: 'delimiter.bracket.js' },
-				{ startIndex: 22, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: '    var    ',
-			tokens: [
-				{ startIndex: 0, scopes: '' },
-				{ startIndex: 4, scopes: 'keyword.js' },
-				{ startIndex: 7, scopes: '' }
-			]}],
-
-			// Comments - single line
-			[{
-			line: '//',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '    // a comment',
-			tokens: [
-				{ startIndex: 0, scopes: '' },
-				{ startIndex: 4, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '// a comment',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '// a comment /*',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '// a comment /**',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '//sticky comment',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: 'var x = 1; // my comment // is a nice one',
-			tokens: [
-				{ startIndex: 0, scopes: 'keyword.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'number.js' },
-				{ startIndex: 9, scopes: 'delimiter.js' },
-				{ startIndex: 10, scopes: '' },
-				{ startIndex: 11, scopes: 'comment.js' }
-			]}],
-
-			// Comments - range comment, single line
-			[{
-			line: '/* a simple comment */',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: 'var x = /* a simple comment */ 1;',
-			tokens: [
-				{ startIndex: 0, scopes: 'keyword.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'comment.js' },
-				{ startIndex: 30, scopes: '' },
-				{ startIndex: 31, scopes: 'number.js' },
-				{ startIndex: 32, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: 'x = /**/;',
-			tokens: [
-				{ startIndex: 0, scopes: 'identifier.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'comment.js' },
-				{ startIndex: 8, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: 'x = /*/;',
-			tokens: [
-				{ startIndex: 0, scopes: 'identifier.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'comment.js' }
-			]}],
-
-			// Comments - range comment, multi lines
-			[{
-			line: '/* a multiline comment',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}, {
-			line: 'can actually span',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}, {
-			line: 'multiple lines */',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: 'var x = /* start a comment',
-			tokens: [
-				{ startIndex: 0, scopes: 'keyword.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'comment.js' }
-			]}, {
-			line: ' a ',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}, {
-			line: 'and end it */ var a = 2;',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' },
-				{ startIndex: 13, scopes: '' },
-				{ startIndex: 14, scopes: 'keyword.js' },
-				{ startIndex: 17, scopes: '' },
-				{ startIndex: 18, scopes: 'identifier.js' },
-				{ startIndex: 19, scopes: '' },
-				{ startIndex: 20, scopes: 'delimiter.js' },
-				{ startIndex: 21, scopes: '' },
-				{ startIndex: 22, scopes: 'number.js' },
-				{ startIndex: 23, scopes: 'delimiter.js' }
-			]}],
-
-			// Strings
-			[{
-			line: 'var a = \'a\';',
-			tokens: [
-				{ startIndex: 0, scopes: 'keyword.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'string.js' },
-				{ startIndex: 11, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: '"use strict";',
-			tokens: [
-				{ startIndex: 0, scopes: 'string.js' },
-				{ startIndex: 12, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: 'b = a + " \'cool\'  "',
-			tokens: [
-				{ startIndex: 0, scopes: 'identifier.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'string.js' }
-			]}],
-
-			[{
-			line: '"escaping \\"quotes\\" is cool"',
-			tokens: [
-				{ startIndex: 0, scopes: 'string.js' }
-			]}],
-
-			[{
-			line: '\'\'\'',
-			tokens: [
-				{ startIndex: 0, scopes: 'string.js' }
-			]}],
-
-			[{
-			line: '\'\\\'\'',
-			tokens: [
-				{ startIndex: 0, scopes: 'string.js' }
-			]}],
-
-			[{
-			line: '\'be careful \\not to escape\'',
-			tokens: [
-				{ startIndex: 0, scopes: 'string.js' }
-			]}],
-
-			// Numbers
-			[{
-			line: '0',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: ' 0',
-			tokens: [
-				{ startIndex: 0, scopes: '' },
-				{ startIndex: 1, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: ' 0 ',
-			tokens: [
-				{ startIndex: 0, scopes: '' },
-				{ startIndex: 1, scopes: 'number.js' },
-				{ startIndex: 2, scopes: '' }
-			]}],
-
-			[{
-			line: '0 ',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: '' }
-			]}],
-
-			[{
-			line: '0+0',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: 'delimiter.js' },
-				{ startIndex: 2, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: '100+10',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 3, scopes: 'delimiter.js' },
-				{ startIndex: 4, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: '0 + 0',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: '0123',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: '01239',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' }
-			]}],
-
-			[{
-			line: '0x',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: 'identifier.js' }
-			]}],
-
-			[{
-			line: '0x123',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' }
-			]}],
-
-			// Regular Expressions
-			[{
-			line: '//',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '/**/',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '/***/',
-			tokens: [
-				{ startIndex: 0, scopes: 'comment.doc.js' }
-			]}],
-
-			[{
-			line: '5 / 3;',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'number.js' },
-				{ startIndex: 5, scopes: 'delimiter.js' }
-			]}],
-
-			// Advanced regular expressions
-			[{
-			line: '1 / 2; /* comment',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'number.js' },
-				{ startIndex: 5, scopes: 'delimiter.js' },
-				{ startIndex: 6, scopes: '' },
-				{ startIndex: 7, scopes: 'comment.js' }
-			]}],
-
-			[{
-			line: '1 / 2 / x / b;',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'number.js' },
-				{ startIndex: 5, scopes: '' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'identifier.js' },
-				{ startIndex: 9, scopes: '' },
-				{ startIndex: 10, scopes: 'delimiter.js' },
-				{ startIndex: 11, scopes: '' },
-				{ startIndex: 12, scopes: 'identifier.js' },
-				{ startIndex: 13, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: 'a /ads/ b;',
-			tokens: [
-				{ startIndex: 0, scopes: 'identifier.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: 'identifier.js' },
-				{ startIndex: 6, scopes: 'delimiter.js' },
-				{ startIndex: 7, scopes: '' },
-				{ startIndex: 8, scopes: 'identifier.js' },
-				{ startIndex: 9, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: '1/(2/3)/2/3;',
-			tokens: [
-				{ startIndex: 0, scopes: 'number.js' },
-				{ startIndex: 1, scopes: 'delimiter.js' },
-				{ startIndex: 2, scopes: 'delimiter.parenthesis.js' },
-				{ startIndex: 3, scopes: 'number.js' },
-				{ startIndex: 4, scopes: 'delimiter.js' },
-				{ startIndex: 5, scopes: 'number.js' },
-				{ startIndex: 6, scopes: 'delimiter.parenthesis.js' },
-				{ startIndex: 7, scopes: 'delimiter.js' },
-				{ startIndex: 8, scopes: 'number.js' },
-				{ startIndex: 9, scopes: 'delimiter.js' },
-				{ startIndex: 10, scopes: 'number.js' },
-				{ startIndex: 11, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: '{ key: 123 }',
-			tokens: [
-				{ startIndex: 0, scopes: 'delimiter.bracket.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: 'delimiter.js' },
-				{ startIndex: 6, scopes: '' },
-				{ startIndex: 7, scopes: 'number.js' },
-				{ startIndex: 10, scopes: '' },
-				{ startIndex: 11, scopes: 'delimiter.bracket.js' }
-			]}],
-
-			[{
-			line: '[1,2,3]',
-			tokens: [
-				{ startIndex: 0, scopes: 'delimiter.array.js' },
-				{ startIndex: 1, scopes: 'number.js' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: 'number.js' },
-				{ startIndex: 4, scopes: 'delimiter.js' },
-				{ startIndex: 5, scopes: 'number.js' },
-				{ startIndex: 6, scopes: 'delimiter.array.js' }
-			]}],
-
-			[{
-			line: 'foo(123);',
-			tokens: [
-				{ startIndex: 0, scopes: 'identifier.js' },
-				{ startIndex: 3, scopes: 'delimiter.parenthesis.js' },
-				{ startIndex: 4, scopes: 'number.js' },
-				{ startIndex: 7, scopes: 'delimiter.parenthesis.js' },
-				{ startIndex: 8, scopes: 'delimiter.js' }
-			]}],
-
-			[{
-			line: '{a:{b:[]}}',
-			tokens: [
-				{ startIndex: 0, scopes: 'delimiter.bracket.js' },
-				{ startIndex: 1, scopes: 'identifier.js' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: 'delimiter.bracket.js' },
-				{ startIndex: 4, scopes: 'identifier.js' },
-				{ startIndex: 5, scopes: 'delimiter.js' },
-				{ startIndex: 6, scopes: 'delimiter.array.js' },
-				{ startIndex: 8, scopes: 'delimiter.bracket.js' }
-			]}],
-
-			[{
-			line: 'x = "[{()}]"',
-			tokens: [
-				{ startIndex: 0, scopes: 'identifier.js' },
-				{ startIndex: 1, scopes: '' },
-				{ startIndex: 2, scopes: 'delimiter.js' },
-				{ startIndex: 3, scopes: '' },
-				{ startIndex: 4, scopes: 'string.js' }
-			]}]
-		]);
-	});
-
-})

src/tokenization.ts

@@ -1,163 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import * as ts from './lib/typescriptServices';
-
-export enum Language {
-	TypeScript,
-	EcmaScript5
-}
-
-export function createTokenizationSupport(language: Language): monaco.languages.TokensProvider {
-
-	var classifier = ts.createClassifier(),
-		bracketTypeTable = language === Language.TypeScript ? tsBracketTypeTable : jsBracketTypeTable,
-		tokenTypeTable = language === Language.TypeScript ? tsTokenTypeTable : jsTokenTypeTable;
-
-	return {
-		getInitialState: () => new State(language, ts.EndOfLineState.None, false),
-		tokenize: (line, state) => tokenize(bracketTypeTable, tokenTypeTable, classifier, <State>state, line)
-	};
-}
-
-class State implements monaco.languages.IState {
-
-	public language: Language;
-	public eolState: ts.EndOfLineState;
-	public inJsDocComment: boolean;
-
-	constructor(language: Language, eolState: ts.EndOfLineState, inJsDocComment: boolean) {
-		this.language = language;
-		this.eolState = eolState;
-		this.inJsDocComment = inJsDocComment;
-	}
-
-	public clone(): State {
-		return new State(this.language, this.eolState, this.inJsDocComment);
-	}
-
-	public equals(other: monaco.languages.IState): boolean {
-		if (other === this) {
-			return true;
-		}
-		if (!other || !(other instanceof State)) {
-			return false;
-		}
-		if (this.eolState !== (<State>other).eolState) {
-			return false;
-		}
-		if (this.inJsDocComment !== (<State>other).inJsDocComment) {
-			return false;
-		}
-		return true;
-	}
-}
-
-function tokenize(bracketTypeTable: { [i: number]: string }, tokenTypeTable: { [i: number]: string },
-	classifier: ts.Classifier, state: State, text: string): monaco.languages.ILineTokens {
-
-	// Create result early and fill in tokens
-	var ret = {
-		tokens: <monaco.languages.IToken[]>[],
-		endState: new State(state.language, ts.EndOfLineState.None, false)
-	};
-
-	function appendFn(startIndex: number, type: string): void {
-		if (ret.tokens.length === 0 || ret.tokens[ret.tokens.length - 1].scopes !== type) {
-			ret.tokens.push({
-				startIndex: startIndex,
-				scopes: type
-			});
-		}
-	}
-
-	var isTypeScript = state.language === Language.TypeScript;
-
-	// shebang statement, #! /bin/node
-	if (!isTypeScript && checkSheBang(0, text, appendFn)) {
-		return ret;
-	}
-
-	var result = classifier.getClassificationsForLine(text, state.eolState, true),
-		offset = 0;
-
-	ret.endState.eolState = result.finalLexState;
-	ret.endState.inJsDocComment = result.finalLexState === ts.EndOfLineState.InMultiLineCommentTrivia && (state.inJsDocComment || /\/\*\*.*$/.test(text));
-
-	for (let entry of result.entries) {
-
-		var type: string;
-
-		if (entry.classification === ts.TokenClass.Punctuation) {
-			// punctions: check for brackets: (){}[]
-			var ch = text.charCodeAt(offset);
-			type = bracketTypeTable[ch] || tokenTypeTable[entry.classification];
-			appendFn(offset, type);
-
-		} else if (entry.classification === ts.TokenClass.Comment) {
-			// comments: check for JSDoc, block, and line comments
-			if (ret.endState.inJsDocComment || /\/\*\*.*\*\//.test(text.substr(offset, entry.length))) {
-				appendFn(offset, isTypeScript ? 'comment.doc.ts' : 'comment.doc.js');
-			} else {
-				appendFn(offset, isTypeScript ? 'comment.ts' : 'comment.js');
-			}
-		} else {
-			// everything else
-			appendFn(offset,
-				tokenTypeTable[entry.classification] || '');
-		}
-
-		offset += entry.length;
-	}
-
-	return ret;
-}
-
-interface INumberStringDictionary {
-	[idx: number]: string;
-}
-
-var tsBracketTypeTable: INumberStringDictionary = Object.create(null);
-tsBracketTypeTable['('.charCodeAt(0)] = 'delimiter.parenthesis.ts';
-tsBracketTypeTable[')'.charCodeAt(0)] = 'delimiter.parenthesis.ts';
-tsBracketTypeTable['{'.charCodeAt(0)] = 'delimiter.bracket.ts';
-tsBracketTypeTable['}'.charCodeAt(0)] = 'delimiter.bracket.ts';
-tsBracketTypeTable['['.charCodeAt(0)] = 'delimiter.array.ts';
-tsBracketTypeTable[']'.charCodeAt(0)] = 'delimiter.array.ts';
-
-var tsTokenTypeTable: INumberStringDictionary = Object.create(null);
-tsTokenTypeTable[ts.TokenClass.Identifier] = 'identifier.ts';
-tsTokenTypeTable[ts.TokenClass.Keyword] = 'keyword.ts';
-tsTokenTypeTable[ts.TokenClass.Operator] = 'delimiter.ts';
-tsTokenTypeTable[ts.TokenClass.Punctuation] = 'delimiter.ts';
-tsTokenTypeTable[ts.TokenClass.NumberLiteral] = 'number.ts';
-tsTokenTypeTable[ts.TokenClass.RegExpLiteral] = 'regexp.ts';
-tsTokenTypeTable[ts.TokenClass.StringLiteral] = 'string.ts';
-
-var jsBracketTypeTable: INumberStringDictionary = Object.create(null);
-jsBracketTypeTable['('.charCodeAt(0)] = 'delimiter.parenthesis.js';
-jsBracketTypeTable[')'.charCodeAt(0)] = 'delimiter.parenthesis.js';
-jsBracketTypeTable['{'.charCodeAt(0)] = 'delimiter.bracket.js';
-jsBracketTypeTable['}'.charCodeAt(0)] = 'delimiter.bracket.js';
-jsBracketTypeTable['['.charCodeAt(0)] = 'delimiter.array.js';
-jsBracketTypeTable[']'.charCodeAt(0)] = 'delimiter.array.js';
-
-var jsTokenTypeTable: INumberStringDictionary = Object.create(null);
-jsTokenTypeTable[ts.TokenClass.Identifier] = 'identifier.js';
-jsTokenTypeTable[ts.TokenClass.Keyword] = 'keyword.js';
-jsTokenTypeTable[ts.TokenClass.Operator] = 'delimiter.js';
-jsTokenTypeTable[ts.TokenClass.Punctuation] = 'delimiter.js';
-jsTokenTypeTable[ts.TokenClass.NumberLiteral] = 'number.js';
-jsTokenTypeTable[ts.TokenClass.RegExpLiteral] = 'regexp.js';
-jsTokenTypeTable[ts.TokenClass.StringLiteral] = 'string.js';
-
-
-function checkSheBang(deltaOffset: number, line: string, appendFn: (startIndex: number, type: string) => void): boolean {
-	if (line.indexOf('#!') === 0) {
-		appendFn(deltaOffset, 'comment.shebang');
-		return true;
-	}
-}

src/tsMode.ts

@@ -4,7 +4,6 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { Language, createTokenizationSupport } from './tokenization';
 import { WorkerManager } from './workerManager';
 import { TypeScriptWorker } from './tsWorker';
 import { LanguageServiceDefaultsImpl } from './monaco.contribution';
@@ -19,16 +18,14 @@ let typeScriptWorker: (first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker>;
 export function setupTypeScript(defaults: LanguageServiceDefaultsImpl): void {
 	typeScriptWorker = setupMode(
 		defaults,
-		'typescript',
-		Language.TypeScript
+		'typescript'
 	);
 }
 
 export function setupJavaScript(defaults: LanguageServiceDefaultsImpl): void {
 	javaScriptWorker = setupMode(
 		defaults,
-		'javascript',
-		Language.EcmaScript5
+		'javascript'
 	);
 }
 
@@ -52,7 +49,7 @@ export function getTypeScriptWorker(): Promise<(first: Uri, ...more: Uri[]) => P
 	});
 }
 
-function setupMode(defaults: LanguageServiceDefaultsImpl, modeId: string, language: Language): (first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker> {
+function setupMode(defaults: LanguageServiceDefaultsImpl, modeId: string): (first: Uri, ...more: Uri[]) => Promise<TypeScriptWorker> {
 
 	const client = new WorkerManager(modeId, defaults);
 	const worker = (first: Uri, ...more: Uri[]): Promise<TypeScriptWorker> => {
@@ -69,64 +66,6 @@ function setupMode(defaults: LanguageServiceDefaultsImpl, modeId: string, langua
 	monaco.languages.registerDocumentRangeFormattingEditProvider(modeId, new languageFeatures.FormatAdapter(worker));
 	monaco.languages.registerOnTypeFormattingEditProvider(modeId, new languageFeatures.FormatOnTypeAdapter(worker));
 	new languageFeatures.DiagnostcsAdapter(defaults, modeId, worker);
-	monaco.languages.setLanguageConfiguration(modeId, richEditConfiguration);
-	monaco.languages.setTokensProvider(modeId, createTokenizationSupport(language));
 
 	return worker;
 }
-
-const richEditConfiguration: monaco.languages.LanguageConfiguration = {
-	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
-
-	comments: {
-		lineComment: '//',
-		blockComment: ['/*', '*/']
-	},
-
-	brackets: [
-		['{', '}'],
-		['[', ']'],
-		['(', ')']
-	],
-
-	onEnterRules: [
-		{
-			// e.g. /** | */
-			beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
-			afterText: /^\s*\*\/$/,
-			action: { indentAction: monaco.languages.IndentAction.IndentOutdent, appendText: ' * ' }
-		},
-		{
-			// e.g. /** ...|
-			beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
-			action: { indentAction: monaco.languages.IndentAction.None, appendText: ' * ' }
-		},
-		{
-			// e.g.  * ...|
-			beforeText: /^(\t|(\ \ ))*\ \*(\ ([^\*]|\*(?!\/))*)?$/,
-			action: { indentAction: monaco.languages.IndentAction.None, appendText: '* ' }
-		},
-		{
-			// e.g.  */|
-			beforeText: /^(\t|(\ \ ))*\ \*\/\s*$/,
-			action: { indentAction: monaco.languages.IndentAction.None, removeText: 1 }
-		}
-	],
-
-	autoClosingPairs: [
-		{ open: '{', close: '}' },
-		{ open: '[', close: ']' },
-		{ open: '(', close: ')' },
-		{ open: '"', close: '"', notIn: ['string'] },
-		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
-		{ open: '`', close: '`', notIn: ['string', 'comment'] },
-		{ open: "/**", close: " */", notIn: ["string"] }
-	],
-
-	folding: {
-		markers: {
-			start: new RegExp("^\\s*//\\s*#?region\\b"),
-			end: new RegExp("^\\s*//\\s*#?endregion\\b")
-		}
-	}
-};

test/all.js

@@ -1,19 +0,0 @@
-var requirejs = require("requirejs");
-var path = require('path');
-
-requirejs.config({
-    baseUrl: 'release/dev',
-    paths: {
-        'vs/language/typescript': path.join(__dirname, '/../release/dev')
-    },
-    nodeRequire: require
-});
-
-// Workaround for TypeScript
-process.browser = true;
-
-requirejs([
-    'vs/language/typescript/test/tokenization.test'
-], function () {
-    run(); // We can launch the tests!
-});

test/index.html

@@ -12,6 +12,7 @@
 
 <script>
 	var paths = {
+		'vs/basic-languages': '../node_modules/monaco-languages/release/dev',
 		'vs/language/typescript': '../release/dev',
 		'vs': '../node_modules/monaco-editor-core/dev/vs'
 	};
@@ -33,6 +34,7 @@
 
 <script>
 	require([
+		'vs/basic-languages/monaco.contribution',
 		'vs/language/typescript/monaco.contribution'
 	], function() {
 		var editor = monaco.editor.create(document.getElementById('container'), {

test/mocha.opts

@@ -1,3 +0,0 @@
---delay
---ui tdd
-test/all.js