Merge remote-tracking branch 'origin/master' into pr/ALANVF/98

.npmignore

@@ -8,3 +8,5 @@
 /tsconfig.json
 /.npmignore
 /.travis.yml
+/.editorconfig
+/azure-pipelines.yml

.vscode/launch.json

@@ -5,9 +5,11 @@
             "name": "Unit Tests",
             "type": "node",
             "request": "launch",
-            "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
+            "program": "${workspaceRoot}/node_modules/.bin/tape",
             "stopOnEntry": false,
             "args": [
+                "-r",
+                "./test/all.js",
                 // "--grep",
                 // "typescript"
             ],

package.json

@@ -5,7 +5,8 @@
   "scripts": {
     "compile": "mrmdir ./release && tsc -p ./src/tsconfig.json && tsc -p ./src/tsconfig.esm.json",
     "watch": "tsc -p ./src --watch",
-    "test": "mocha",
+    "watch-esm": "tsc -p ./src/tsconfig.esm.json --watch",
+    "test": "tape -r ./test/all.js",
     "prepublishOnly": "npm run compile && node ./scripts/bundle"
   },
   "author": "Microsoft Corporation",
@@ -18,13 +19,14 @@
     "url": "https://github.com/Microsoft/monaco-languages/issues"
   },
   "devDependencies": {
+    "@types/tape": "^4.2.34",
     "glob": "^7.1.6",
-    "jsdom": "^16.1.0",
-    "mocha": "^7.0.1",
+    "jsdom": "^16.4.0",
     "monaco-editor-core": "0.20.0",
     "monaco-plugin-helpers": "^1.0.2",
     "requirejs": "^2.3.6",
-    "terser": "^4.6.3",
+    "tape": "^4.13.2",
+    "terser": "^4.6.6",
     "typescript": "3.7.5"
   }
 }

src/abap/abap.test.ts

@@ -92,4 +92,93 @@ testTokenization('abap', [
 			{ startIndex: 0, type: 'string.abap' },
 		]
 	}],
+	[{
+		line: 'FIELD-SYMBOLS <foo>.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.abap' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'identifier.abap' },
+			{ startIndex: 19, type: 'delimiter.abap' },
+		]
+	}],
+	[{
+		line: 'IF foo IS NOT INITIAL.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.abap' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'identifier.abap' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'keyword.abap' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'keyword.abap' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.abap' },
+			{ startIndex: 21, type: 'delimiter.abap' },
+		]
+	}],
+	[{
+		line: 'WRITE `moo`.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.abap' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'string.abap' },
+			{ startIndex: 11, type: 'delimiter.abap' },
+		]
+	}],
+	[{
+		line: 'FORM foo.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.abap' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'identifier.abap' },
+			{ startIndex: 8, type: 'delimiter.abap' },
+		]
+	}],
+	[{
+		line: 'moo = CONV #( 1 ).',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.abap' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'operator.abap' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'keyword.abap' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'operator.abap' },
+			{ startIndex: 12, type: "delimiter.parenthesis.abap" },
+			{ startIndex: 13, type: "" },
+			{ startIndex: 14, type: "number.abap" },
+			{ startIndex: 15, type: "" },
+			{ startIndex: 16, type: "delimiter.parenthesis.abap" },
+			{ startIndex: 17, type: "delimiter.abap" }
+		]
+	}],
+	[{
+		line: 'WRITE foo ##pragma.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.abap' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'identifier.abap' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 12, type: 'identifier.abap' },
+			{ startIndex: 18, type: 'delimiter.abap' },
+		]
+	}],
+	[{
+		line: 'SELECT * FROM foo02 INTO @foo.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.abap' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'operator.abap' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'keyword.abap' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'identifier.abap' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'keyword.abap' },
+			{ startIndex: 24, type: '' },
+			{ startIndex: 25, type: 'operator.abap' },
+			{ startIndex: 26, type: 'identifier.abap' },
+			{ startIndex: 29, type: 'delimiter.abap' },
+		]
+	}],
 ]);

src/abap/abap.ts

@@ -29,26 +29,27 @@ const abapKeywords = [
 	'editor-call','end','endexec','endfunction','ending','endmodule','end-of-definition','end-of-page','end-of-selection','end-test-injection','end-test-seam','exit-command','endclass','endmethod','endform','endinterface',
 	'endprovide','endselect','endtry','endwhile','enum','event','events','exec','exit','export',
 	'exporting','extract','exception','exceptions',
-	'field-symbols','field-groups','field','first','fetch','fields','format','frame','free','from','function','find','for','found','function-pool',
+	'form','field-symbols','field-groups','field','first','fetch','fields','format','frame','free','from','function','find','for','found','function-pool',
 	'generate','get',
 	'handle','hide','hashed',
 	'include','import','importing','index','infotypes','initial','initialization',
-	'id','is','in','interface','interfaces','init','input','insert','instance','into',
+	'id','is','in','interface','interfaces','init','input','insert','instance','into','implemented',
 	'key',
 	'left-justified','leave','like','line','line-count','line-size','load','local','log-point','length','left','leading','lower',
-	'matchcode','method','mesh','message','message-id','methods','modify','module','move','move-corresponding','multiply','multiply-corresponding','match',
-	'new','new-line','new-page','new-section','next','no','no-gap','no-gaps','no-sign','no-zero','non-unique','number',
+	'matchcode','method','mesh','message','message-id','methods','modify','module','move','move-corresponding','multiply','multiply-corresponding','match','mode',
+	'not','new','new-line','new-page','new-section','next','no','no-gap','no-gaps','no-sign','no-zero','non-unique','number',
 	'occurrence','object','obligatory','of','output','overlay','optional','others','occurrences','occurs','offset','options',
-	'pack','parameters','perform','places','position','print-control','private','program','protected','provide','public','put',
+	'pack','parameters','perform','places','position','print-control','private','program','protected','provide','public','put','partially',
 	'radiobutton','raising','ranges','receive','receiving','redefinition','reduce','reference','refresh','regex','reject','results','requested',
 	'ref','replace','report','reserve','restore','result','return','returning','right-justified','rollback','read','read-only','rp-provide-from-last','run',
-	'scan','screen','scroll','search','select','select-options','selection-screen','stamp','source','subkey',
+	'scan','screen','scroll','search','select','select-options','selection-screen','stamp','source','subkey','subscreen',
 	'separated','set','shift','single','skip','sort','sorted','split','standard','stamp','starting','start-of-selection','sum','subtract-corresponding','statics','step','stop','structure','submatches','submit','subtract','summary','supplied','suppress','section','syntax-check','syntax-trace','system-call','switch',
-	'tables','table','task','testing','test-seam','test-injection','then','time','times','title','titlebar','to','top-of-page','trailing','transfer','transformation','translate','transporting','types','type','type-pool','type-pools',
-	'unassign','unique','uline','unpack','update','upper','using',
+	'tables','table','task','testing','test-seam','test-injection','then','time','times','title','titlebar','to','top-of-page','trailing','transfer','transformation','translate','transporting','types','type','type-pool','type-pools','tabbed',
+	'unassign','unique','uline','unpack','update','upper','using','user-command',
 	'value',
 	'when','while','window','write','where','with','work',
 	'at','case','catch','continue','do','elseif','else','endat','endcase','enddo','endif','endloop','endon','if','loop','on','raise','try',
+	// built-in:
 	'abs','sign','ceil','floor','trunc','frac','acos','asin','atan','cos','sin','tan','cosh','sinh','tanh','exp','log','log10','sqrt','strlen','xstrlen','charlen','lines','numofchar','dbmaxlen','round','rescale','nmax','nmin','cmax','cmin','boolc','boolx','xsdbool','contains','contains_any_of','contains_any_not_of','matches','line_exists','ipow','char_off','count','count_any_of','count_any_not_of','distance','condense','concat_lines_of','escape','find','find_end','find_any_of','find_any_not_of','insert','match','repeat','replace','reverse','segment','shift_left','shift_right','substring','substring_after','substring_from','substring_before','substring_to','to_upper','to_lower','to_mixed','from_mixed','translate','bit-set','line_index',
 	'definition','implementation','public','inheriting','final'
   ];
@@ -65,19 +66,22 @@ export const language = <ILanguage> {
 	],
 
 	operators: [
-	  '+', '-', '/', '*',
-	  '=', '<', '>', '<=', '>=', '<>', '><', '=<', '=>',
+	  ' +', ' -', '/', '*',
+	  '=', ' < ', ' > ', '<=', '>=', '<>', '><', '=<', '=>',
+	  '#', '@',
 	  'EQ', 'NE', 'GE', 'LE',
 	  'CS', 'CN', 'CA', 'CO', 'CP', 'NS', 'NA', 'NP',
 	],
 
-	symbols:  /[=><!~?&+\-*\/\^%]+/,
+	symbols:  /[=><!~?&+\-*\/\^%#@]+/,
 
 	tokenizer: {
 	  root: [
-		[/[a-z_$][\w$]*/, { cases: { '@typeKeywords': 'keyword',
-									 '@keywords': 'keyword',
-									 '@default': 'identifier' } }],
+		[/[a-z_$][\w-$]*/, { cases: { '@typeKeywords': 'keyword',
+								 	  '@keywords': 'keyword',
+							 		  '@default': 'identifier' } }],
+
+		[/<[\w]+>/, 'identifier'], // field symbols
 
 		{ include: '@whitespace' },
 
@@ -88,6 +92,7 @@ export const language = <ILanguage> {
 								'@default'  : '' } } ],
 
 		[/'/,  { token: 'string', bracket: '@open', next: '@stringquote' } ],
+		[/`/,  { token: 'string', bracket: '@open', next: '@stringping' } ],
 		[/\|/,  { token: 'string', bracket: '@open', next: '@stringtemplate' } ],
 
 		[/\d+/, 'number'],
@@ -99,6 +104,11 @@ export const language = <ILanguage> {
 		[/\|/,     { token: 'string', bracket: '@close', next: '@pop' } ]
 	  ],
 
+	  stringping: [
+		[/[^\\`]+/, 'string'],
+		[/`/,       { token: 'string', bracket: '@close', next: '@pop' } ]
+	  ],
+
 	  stringquote: [
 		[/[^\\']+/, 'string'],
 		[/'/,       { token: 'string', bracket: '@close', next: '@pop' } ]

src/dart/dart.contribution.ts

@@ -0,0 +1,15 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'dart',
+	extensions: ['.dart'],
+	aliases: ['Dart', 'dart'],
+	mimetypes: ['text/x-dart-source', 'text/x-dart'],
+	loader: () => import('./dart')
+});

src/dart/dart.test.ts

@@ -0,0 +1,559 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+"use strict";
+
+import { testTokenization } from "../test/testRunner";
+
+testTokenization("dart", [
+	// Comments - single line
+	[
+		{
+			line: "//",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "    // a comment",
+			tokens: [
+				{ startIndex: 0, type: "" },
+				{ startIndex: 4, type: "comment.dart" }
+			]
+		}
+	],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[
+		{
+			line: "/* //*/ a",
+			tokens: [
+				{ startIndex: 0, type: "comment.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "// a comment",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "//sticky comment",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "/almost a comment",
+			tokens: [
+				{ startIndex: 0, type: "delimiter.dart" },
+				{ startIndex: 1, type: "identifier.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "identifier.dart" },
+				{ startIndex: 9, type: "" },
+				{ startIndex: 10, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "1 / 2; /* comment",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "" },
+				{ startIndex: 2, type: "delimiter.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "number.dart" },
+				{ startIndex: 5, type: "delimiter.dart" },
+				{ startIndex: 6, type: "" },
+				{ startIndex: 7, type: "comment.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "var x = 1; // my comment // is a nice one",
+			tokens: [
+				{ startIndex: 0, type: "keyword.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "identifier.dart" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "delimiter.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "number.dart" },
+				{ startIndex: 9, type: "delimiter.dart" },
+				{ startIndex: 10, type: "" },
+				{ startIndex: 11, type: "comment.dart" }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: "/* a simple comment */",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "var x = /* a simple comment */ 1;",
+			tokens: [
+				{ startIndex: 0, type: "keyword.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "identifier.dart" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "delimiter.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "comment.dart" },
+				{ startIndex: 30, type: "" },
+				{ startIndex: 31, type: "number.dart" },
+				{ startIndex: 32, type: "delimiter.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "var x = /* comment */ 1; */",
+			tokens: [
+				{ startIndex: 0, type: "keyword.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "identifier.dart" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "delimiter.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "comment.dart" },
+				{ startIndex: 21, type: "" },
+				{ startIndex: 22, type: "number.dart" },
+				{ startIndex: 23, type: "delimiter.dart" },
+				{ startIndex: 24, type: "" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "x = /**/;",
+			tokens: [
+				{ startIndex: 0, type: "identifier.dart" },
+				{ startIndex: 1, type: "" },
+				{ startIndex: 2, type: "delimiter.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "comment.dart" },
+				{ startIndex: 8, type: "delimiter.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "x = /*/;",
+			tokens: [
+				{ startIndex: 0, type: "identifier.dart" },
+				{ startIndex: 1, type: "" },
+				{ startIndex: 2, type: "delimiter.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "comment.dart" }
+			]
+		}
+	],
+
+	// Comments - range comment, multiple lines
+	[
+		{
+			line: "/* start of multiline comment",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		},
+		{
+			line: "a comment between without a star",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		},
+		{
+			line: "end of multiline comment*/",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "var x = /* start a comment",
+			tokens: [
+				{ startIndex: 0, type: "keyword.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "identifier.dart" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "delimiter.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "comment.dart" }
+			]
+		},
+		{
+			line: " a ",
+			tokens: [{ startIndex: 0, type: "comment.dart" }]
+		},
+		{
+			line: "and end it */ 2;",
+			tokens: [
+				{ startIndex: 0, type: "comment.dart" },
+				{ startIndex: 13, type: "" },
+				{ startIndex: 14, type: "number.dart" },
+				{ startIndex: 15, type: "delimiter.dart" }
+			]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: "var x = function() { };",
+			tokens: [
+				{ startIndex: 0, type: "keyword.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "identifier.dart" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "delimiter.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "identifier.dart" },
+				{ startIndex: 16, type: "delimiter.parenthesis.dart" },
+				{ startIndex: 18, type: "" },
+				{ startIndex: 19, type: "delimiter.bracket.dart" },
+				{ startIndex: 20, type: "" },
+				{ startIndex: 21, type: "delimiter.bracket.dart" },
+				{ startIndex: 22, type: "delimiter.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "    var    ",
+			tokens: [
+				{ startIndex: 0, type: "" },
+				{ startIndex: 4, type: "keyword.dart" },
+				{ startIndex: 7, type: "" }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: "0",
+			tokens: [{ startIndex: 0, type: "number.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "0.10",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "0x",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "0x123",
+			tokens: [{ startIndex: 0, type: "number.hex.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "0x5_2",
+			tokens: [{ startIndex: 0, type: "number.hex.dart" }]
+		}
+	],
+	[
+		{
+			line: "0b1010_0101",
+			tokens: [{ startIndex: 0, type: "number.binary.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "0B001",
+			tokens: [{ startIndex: 0, type: "number.binary.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "10e3",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+	[
+		{
+			line: "23.5",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "23.5e3",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "23.5e-3",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "23.5E3",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "23.5E-3",
+			tokens: [{ startIndex: 0, type: "number.float.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "0_52",
+			tokens: [{ startIndex: 0, type: "number.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "5_2",
+			tokens: [{ startIndex: 0, type: "number.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "5_______2",
+			tokens: [{ startIndex: 0, type: "number.dart" }]
+		}
+	],
+	[
+		{
+			line: "3._1415F",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "delimiter.dart" },
+				{ startIndex: 2, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "999_99_9999_L",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 11, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "52_",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 2, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "0_x52",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "0x_52",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "0x52_",
+			tokens: [
+				{ startIndex: 0, type: "number.hex.dart" },
+				{ startIndex: 4, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "052_",
+			tokens: [
+				{ startIndex: 0, type: "number.octal.dart" },
+				{ startIndex: 3, type: "identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "23.5L",
+			tokens: [
+				{ startIndex: 0, type: "number.float.dart" },
+				{ startIndex: 4, type: "type.identifier.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "0+0",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "delimiter.dart" },
+				{ startIndex: 2, type: "number.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "100+10",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 3, type: "delimiter.dart" },
+				{ startIndex: 4, type: "number.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: "0 + 0",
+			tokens: [
+				{ startIndex: 0, type: "number.dart" },
+				{ startIndex: 1, type: "" },
+				{ startIndex: 2, type: "delimiter.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "number.dart" }
+			]
+		}
+	],
+
+	// Strings
+	[
+		{
+			line: "var s = 's';",
+			tokens: [
+				{ startIndex: 0, type: "keyword.dart" },
+				{ startIndex: 3, type: "" },
+				{ startIndex: 4, type: "identifier.dart" },
+				{ startIndex: 5, type: "" },
+				{ startIndex: 6, type: "delimiter.dart" },
+				{ startIndex: 7, type: "" },
+				{ startIndex: 8, type: "string.dart" },
+				{ startIndex: 11, type: "delimiter.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = "concatenated" + " String" ;',
+			tokens: [
+				{ startIndex: 0, type: "type.identifier.dart" },
+				{ startIndex: 6, type: "" },
+				{ startIndex: 7, type: "identifier.dart" },
+				{ startIndex: 8, type: "" },
+				{ startIndex: 9, type: "delimiter.dart" },
+				{ startIndex: 10, type: "" },
+				{ startIndex: 11, type: "string.dart" },
+				{ startIndex: 25, type: "" },
+				{ startIndex: 26, type: "delimiter.dart" },
+				{ startIndex: 27, type: "" },
+				{ startIndex: 28, type: "string.dart" },
+				{ startIndex: 37, type: "" },
+				{ startIndex: 38, type: "delimiter.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"quote in a string"',
+			tokens: [{ startIndex: 0, type: "string.dart" }]
+		}
+	],
+
+	[
+		{
+			line: '"escaping \\"quotes\\" is cool"',
+			tokens: [
+				{ startIndex: 0, type: "string.dart" },
+				{ startIndex: 10, type: "string.escape.dart" },
+				{ startIndex: 12, type: "string.dart" },
+				{ startIndex: 18, type: "string.escape.dart" },
+				{ startIndex: 20, type: "string.dart" }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"\\"',
+			tokens: [{ startIndex: 0, type: "string.invalid.dart" }]
+		}
+	],
+
+	// Annotations
+	[
+		{
+			line: "@",
+			tokens: [{ startIndex: 0, type: "invalid.dart" }]
+		}
+	],
+
+	[
+		{
+			line: "@Override",
+			tokens: [{ startIndex: 0, type: "annotation.dart" }]
+		}
+	]
+]);

src/dart/dart.ts

@@ -0,0 +1,326 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+"use strict";
+
+import LanguageConfiguration = monaco.languages.LanguageConfiguration;
+import IMonarchLanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: LanguageConfiguration = {
+	comments: {
+		lineComment: "//",
+		blockComment: ["/*", "*/"]
+	},
+	brackets: [
+		["{", "}"],
+		["[", "]"],
+		["(", ")"]
+	],
+	autoClosingPairs: [
+		{ open: "{", close: "}" },
+		{ open: "[", close: "]" },
+		{ open: "(", close: ")" },
+		{ open: "'", close: "'", notIn: ["string", "comment"] },
+		{ open: '"', close: '"', notIn: ["string"] },
+		{ open: "`", close: "`", notIn: ["string", "comment"] },
+		{ open: "/**", close: " */", notIn: ["string"] }
+	],
+	surroundingPairs: [
+		{ open: "{", close: "}" },
+		{ open: "[", close: "]" },
+		{ open: "(", close: ")" },
+
+		{ open: "<", close: ">" },
+		{ open: "'", close: "'" },
+		{ open: "(", close: ")" },
+		{ open: '"', close: '"' },
+		{ open: "`", close: "`" }
+	],
+	folding: {
+		markers: {
+			start: /^\s*\s*#?region\b/,
+			end: /^\s*\s*#?endregion\b/
+		}
+	}
+};
+
+export const language = <IMonarchLanguage>{
+	defaultToken: "invalid",
+	tokenPostfix: ".dart",
+
+	keywords: [
+		"abstract",
+		"dynamic",
+		"implements",
+		"show",
+		"as",
+		"else",
+		"import",
+		"static",
+		"assert",
+		"enum",
+		"in",
+		"super",
+		"async",
+		"export",
+		"interface",
+		"switch",
+		"await",
+		"extends",
+		"is",
+		"sync",
+		"break",
+		"external",
+		"library",
+		"this",
+		"case",
+		"factory",
+		"mixin",
+		"throw",
+		"catch",
+		"false",
+		"new",
+		"true",
+		"class",
+		"final",
+		"null",
+		"try",
+		"const",
+		"finally",
+		"on",
+		"typedef",
+		"continue",
+		"for",
+		"operator",
+		"var",
+		"covariant",
+		"Function",
+		"part",
+		"void",
+		"default",
+		"get",
+		"rethrow",
+		"while",
+		"deferred",
+		"hide",
+		"return",
+		"with",
+		"do",
+		"if",
+		"set",
+		"yield"
+	],
+	typeKeywords: ["int", "double", "String", "bool"],
+
+	operators: [
+		"+",
+		"-",
+		"*",
+		"/",
+		"~/",
+		"%",
+		"++",
+		"--",
+		"==",
+		"!=",
+		">",
+		"<",
+		">=",
+		"<=",
+		"=",
+		"-=",
+		"/=",
+		"%=",
+		">>=",
+		"^=",
+		"+=",
+		"*=",
+		"~/=",
+		"<<=",
+		"&=",
+		"!=",
+		"||",
+		"&&",
+		"&",
+		"|",
+		"^",
+		"~",
+		"<<",
+		">>",
+		"!",
+		">>>",
+		"??",
+		"?",
+		":",
+		"|="
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	regexpctl: /[(){}\[\]\$\^|\-*+?\.]/,
+	regexpesc: /\\(?:[bBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [[/[{}]/, "delimiter.bracket"], { include: "common" }],
+
+		common: [
+			// identifiers and keywords
+			[
+				/[a-z_$][\w$]*/,
+				{
+					cases: {
+						"@typeKeywords": "type.identifier",
+						"@keywords": "keyword",
+						"@default": "identifier"
+					}
+				}
+			],
+			[
+				/(?<![a-zA-Z0-9_$])([_$]*[A-Z][a-zA-Z0-9_$]*)/,
+				"type.identifier"
+			], // to show class names nicely
+			// [/[A-Z][\w\$]*/, 'identifier'],
+
+			// whitespace
+			{ include: "@whitespace" },
+
+			// regular expression: ensure it is terminated before beginning (otherwise it is an opeator)
+			[
+				/\/(?=([^\\\/]|\\.)+\/([gimsuy]*)(\s*)(\.|;|,|\)|\]|\}|$))/,
+				{ token: "regexp", bracket: "@open", next: "@regexp" }
+			],
+
+			// @ annotations.
+			[/@[a-zA-Z]+/, "annotation"],
+			// variable
+
+			// delimiters and operators
+			[/[()\[\]]/, "@brackets"],
+			[/[<>](?!@symbols)/, "@brackets"],
+			[/!(?=([^=]|$))/, "delimiter"],
+			[
+				/@symbols/,
+				{
+					cases: {
+						"@operators": "delimiter",
+						"@default": ""
+					}
+				}
+			],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?/, "number.float"],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?/, "number.float"],
+			[/0[xX](@hexdigits)n?/, "number.hex"],
+			[/0[oO]?(@octaldigits)n?/, "number.octal"],
+			[/0[bB](@binarydigits)n?/, "number.binary"],
+			[/(@digits)n?/, "number"],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, "delimiter"],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, "string.invalid"], // non-teminated string
+			[/'([^'\\]|\\.)*$/, "string.invalid"], // non-teminated string
+			[/"/, "string", "@string_double"],
+			[/'/, "string", "@string_single"]
+
+			//   [/[a-zA-Z]+/, "variable"]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ""],
+			[/\/\*\*(?!\/)/, "comment.doc", "@jsdoc"],
+			[/\/\*/, "comment", "@comment"],
+			[/\/\/\/.*$/, "comment.doc"],
+			[/\/\/.*$/, "comment"]
+		],
+
+		comment: [
+			[/[^\/*]+/, "comment"],
+			[/\*\//, "comment", "@pop"],
+			[/[\/*]/, "comment"]
+		],
+
+		jsdoc: [
+			[/[^\/*]+/, "comment.doc"],
+			[/\*\//, "comment.doc", "@pop"],
+			[/[\/*]/, "comment.doc"]
+		],
+
+		// We match regular expression quite precisely
+		regexp: [
+			[
+				/(\{)(\d+(?:,\d*)?)(\})/,
+				[
+					"regexp.escape.control",
+					"regexp.escape.control",
+					"regexp.escape.control"
+				]
+			],
+			[
+				/(\[)(\^?)(?=(?:[^\]\\\/]|\\.)+)/,
+				[
+					"regexp.escape.control",
+					{ token: "regexp.escape.control", next: "@regexrange" }
+				]
+			],
+			[
+				/(\()(\?:|\?=|\?!)/,
+				["regexp.escape.control", "regexp.escape.control"]
+			],
+			[/[()]/, "regexp.escape.control"],
+			[/@regexpctl/, "regexp.escape.control"],
+			[/[^\\\/]/, "regexp"],
+			[/@regexpesc/, "regexp.escape"],
+			[/\\\./, "regexp.invalid"],
+			[
+				/(\/)([gimsuy]*)/,
+				[
+					{ token: "regexp", bracket: "@close", next: "@pop" },
+					"keyword.other"
+				]
+			]
+		],
+
+		regexrange: [
+			[/-/, "regexp.escape.control"],
+			[/\^/, "regexp.invalid"],
+			[/@regexpesc/, "regexp.escape"],
+			[/[^\]]/, "regexp"],
+			[
+				/\]/,
+				{
+					token: "regexp.escape.control",
+					next: "@pop",
+					bracket: "@close"
+				}
+			]
+		],
+
+		string_double: [
+			[/[^\\"\$]+/, "string"], [/[^\\"]+/, "string"],
+			[/@escapes/, "string.escape"],
+			[/\\./, "string.escape.invalid"],
+			[/"/, "string", "@pop"],
+			[/\$\w+/, 'identifier']
+		],
+
+		string_single: [
+			[/[^\\'\$]+/, "string"],
+			[/@escapes/, "string.escape"],
+			[/\\./, "string.escape.invalid"],
+			[/'/, "string", "@pop"],
+			[/\$\w+/, 'identifier']
+		],
+	}
+};

src/handlebars/handlebars.test.ts

@@ -281,5 +281,35 @@ testTokenization(['handlebars', 'css'], [
 			{ startIndex: 30, type: 'delimiter.handlebars' },
 			{ startIndex: 32, type: '' }
 		]
-	}]
+	}],
+
+	// Block comment
+	[{
+		line: '{{!-- block comment --}}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.block.start.handlebars' },
+			{ startIndex: 5, type: 'comment.content.handlebars' },
+			{ startIndex: 20, type: 'comment.block.end.handlebars' }
+		]
+	}],
+
+	// Block comment with mustache
+	[{
+		line: '{{!-- block comment }} with mustache --}}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.block.start.handlebars' },
+			{ startIndex: 5, type: 'comment.content.handlebars' },
+			{ startIndex: 37, type: 'comment.block.end.handlebars' }
+		]
+	}],
+
+	// Handlebars comment
+	[{
+		line: '{{! comment }}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.start.handlebars' },
+			{ startIndex: 3, type: 'comment.content.handlebars' },
+			{ startIndex: 12, type: 'comment.end.handlebars' }
+		]
+	}],
 ]);

src/handlebars/handlebars.ts

@@ -63,9 +63,11 @@ export const language = <ILanguage>{
 	// The main tokenizer for our languages
 	tokenizer: {
 		root: [
+			[/\{\{!--/, 'comment.block.start.handlebars', '@commentBlock'],
+			[/\{\{!/, 'comment.start.handlebars', '@comment'],
 			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.root' }],
 			[/<!DOCTYPE/, 'metatag.html', '@doctype'],
-			[/<!--/, 'comment.html', '@comment'],
+			[/<!--/, 'comment.html', '@commentHtml'],
 			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
 			[/(<)(script)/, ['delimiter.html', { token: 'tag.html', next: '@script' }]],
 			[/(<)(style)/, ['delimiter.html', { token: 'tag.html', next: '@style' }]],
@@ -83,6 +85,16 @@ export const language = <ILanguage>{
 		],
 
 		comment: [
+			[/\}\}/, 'comment.end.handlebars', '@pop'],
+			[/./, 'comment.content.handlebars']
+		],
+
+		commentBlock: [
+			[/--\}\}/, 'comment.block.end.handlebars', '@pop'],
+			[/./, 'comment.content.handlebars']
+		],
+
+		commentHtml: [
 			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.comment' }],
 			[/-->/, 'comment.html', '@pop'],
 			[/[^-]+/, 'comment.content.html'],

src/javascript/javascript.contribution.ts

@@ -8,7 +8,7 @@ import { registerLanguage } from '../_.contribution';
 
 registerLanguage({
 	id: 'javascript',
-	extensions: ['.js', '.es6', '.jsx'],
+	extensions: ['.js', '.es6', '.jsx', '.mjs'],
 	firstLine: '^#!.*\\bnode',
 	filenames: ['jakefile'],
 	aliases: ['JavaScript', 'javascript', 'js'],

src/lexon/lexon.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'lexon',
+	extensions: ['.lex'],
+	aliases: ['Lexon'],
+	loader: () => import('./lexon')
+});

src/lexon/lexon.test.ts

@@ -0,0 +1,131 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('lexon', [
+	// Tests
+
+	[{
+		line: 'LEX Paid Escrow',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.lexon' },
+			{ startIndex: 3, type: 'white.lexon' },
+			{ startIndex: 4, type: 'identifier.lexon' },
+			{ startIndex: 8, type: 'white.lexon' },
+			{ startIndex: 9, type: 'identifier.lexon' },
+		]
+	}],
+
+	[{
+		line: 'LEXON: 0.2.20',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.lexon' },
+			{ startIndex: 5, type: 'delimiter.lexon' },
+			{ startIndex: 6, type: 'white.lexon' },
+			{ startIndex: 7, type: 'number.semver.lexon' },
+		]
+	}],
+
+	[{
+		line: 'COMMENT: 3.f - an escrow that is controlled by a third party for a fee.',
+		tokens: [
+			{ startIndex: 0, type: 'comment.lexon' },
+		]
+	}],
+
+	[{
+		line: '"Payer" is a person.',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.lexon' },
+			{ startIndex: 1, type: 'identifier.lexon' },
+			{ startIndex: 6, type: 'identifier.quote.lexon' },
+			{ startIndex: 7, type: 'white.lexon' },
+			{ startIndex: 8, type: 'operator.lexon' },
+			{ startIndex: 10, type: 'white.lexon' },
+			{ startIndex: 11, type: 'identifier.lexon' },
+			{ startIndex: 12, type: 'white.lexon' },
+			{ startIndex: 13, type: 'keyword.type.lexon' },
+			{ startIndex: 19, type: 'delimiter.lexon' },
+		]
+	}],
+
+	[{
+		line: '"Fee" is an amount.',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.lexon' },
+			{ startIndex: 1, type: 'identifier.lexon' },
+			{ startIndex: 4, type: 'identifier.quote.lexon' },
+			{ startIndex: 5, type: 'white.lexon' },
+			{ startIndex: 6, type: 'operator.lexon' },
+			{ startIndex: 8, type: 'white.lexon' },
+			{ startIndex: 9, type: 'identifier.lexon' },
+			{ startIndex: 11, type: 'white.lexon' },
+			{ startIndex: 12, type: 'keyword.type.lexon' },
+			{ startIndex: 18, type: 'delimiter.lexon' },
+		]
+	}],
+
+	[{
+		line: 'The Payer pays an Amount into escrow,',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.lexon' }, 	// The
+			{ startIndex: 3, type: 'white.lexon' },
+			{ startIndex: 4, type: 'identifier.lexon' },	// Payer
+			{ startIndex: 9, type: 'white.lexon' },
+			{ startIndex: 10, type: 'keyword.lexon' },		// pays
+			{ startIndex: 14, type: 'white.lexon' },
+			{ startIndex: 15, type: 'identifier.lexon' },	// an
+			{ startIndex: 17, type: 'white.lexon' },
+			{ startIndex: 18, type: 'keyword.type.lexon' },	// Amount
+			{ startIndex: 24, type: 'white.lexon' },
+			{ startIndex: 25, type: 'keyword.lexon' },		// into
+			{ startIndex: 29, type: 'white.lexon' },
+			{ startIndex: 30, type: 'identifier.lexon' },	// escrow
+			{ startIndex: 36, type: 'delimiter.lexon' },	// ,
+		]
+	}],
+
+	[{
+		line: 'appoints the Payee,',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.lexon' }, 		// Appoints
+			{ startIndex: 8, type: 'white.lexon' },
+			{ startIndex: 9, type: 'identifier.lexon' },	// the
+			{ startIndex: 12, type: 'white.lexon' },
+			{ startIndex: 13, type: 'identifier.lexon' },	// Payee
+			{ startIndex: 18, type: 'delimiter.lexon' },	// ,
+		]
+	}],
+
+	[{
+		line: 'and also fixes the Fee.',
+		tokens: [
+			{ startIndex: 0, type: 'operator.lexon' }, 		// and
+			{ startIndex: 3, type: 'white.lexon' },
+			{ startIndex: 4, type: 'identifier.lexon' },	// also
+			{ startIndex: 8, type: 'white.lexon' },
+			{ startIndex: 9, type: 'identifier.lexon' },	// fixes
+			{ startIndex: 14, type: 'white.lexon' },
+			{ startIndex: 15, type: 'identifier.lexon' },	// the
+			{ startIndex: 18, type: 'white.lexon' },
+			{ startIndex: 19, type: 'identifier.lexon' },	// Fee
+			{ startIndex: 22, type: 'delimiter.lexon' },	// .
+		]
+	}],
+
+	[{
+		line: 'CLAUSE: Pay Out.',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.lexon' }, 		// CLAUSE
+			{ startIndex: 6, type: 'delimiter.lexon' },		// :
+			{ startIndex: 7, type: 'white.lexon' },
+			{ startIndex: 8, type: 'identifier.lexon' },	// Pay out
+			{ startIndex: 15, type: 'delimiter.lexon' },	// .
+		]
+	}],
+]);

src/lexon/lexon.ts

@@ -0,0 +1,140 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	comments: {
+		lineComment: 'COMMENT',
+		// blockComment: ['COMMENT', '.'],
+	},
+	brackets: [
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"', },
+		{ open: ':', close: '.', },
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '`', close: '`' },
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+		{ open: ':', close: '.', },
+	],
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*(::\\s*|COMMENT\\s+)#region"),
+			end: new RegExp("^\\s*(::\\s*|COMMENT\\s+)#endregion")
+		}
+	}
+};
+
+export const language = <ILanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	// defaultToken: 'invalid',
+	tokenPostfix: '.lexon',
+	ignoreCase: true,
+
+	keywords: [
+		'lexon', 'lex', 'clause', 'terms', 'contracts', 'may', 'pay',
+		'pays', 'appoints', 'into', 'to'
+	],
+
+	typeKeywords: [
+		'amount', 'person', 'key', 'time', 'date', 'asset', 'text'
+	],
+
+	operators: [
+		'less', 'greater', 'equal', 'le', 'gt', 'or', 'and',
+		'add', 'added', 'subtract', 'subtracted', 'multiply', 'multiplied', 'times', 'divide', 'divided',
+		'is', 'be', 'certified'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// comment
+			[/^(\s*)(comment:?(?:\s.*|))$/, ['', 'comment']],
+
+			// special identifier cases
+			[/"/, { token: 'identifier.quote', bracket: '@open', next: '@quoted_identifier' }],
+			['LEX$', { token: 'keyword', bracket: '@open', next: '@identifier_until_period' }],
+			['LEXON', { token: 'keyword', bracket: '@open', next: '@semver' }],
+			[':', { token: 'delimiter', bracket: '@open', next: '@identifier_until_period' }],
+
+			// identifiers and keywords
+			[/[a-z_$][\w$]*/, {
+				cases: {
+					'@operators': 'operator',
+					'@typeKeywords': 'keyword.type',
+					'@keywords': 'keyword',
+					'@default': 'identifier'
+				}
+			}],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d*\.\d*\.\d*/, 'number.semver'],
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+		],
+
+		quoted_identifier: [
+			[/[^\\"]+/, 'identifier'],
+			[/"/, { token: 'identifier.quote', bracket: '@close', next: '@pop' }]
+		],
+
+		space_identifier_until_period: [
+			[':', 'delimiter'],
+			[' ', { token: 'white', next: '@identifier_rest' }],
+		],
+
+		identifier_until_period: [
+			{ include: '@whitespace' },
+			[':', { token: 'delimiter', next: '@identifier_rest' }],
+			[/[^\\.]+/, 'identifier'],
+			[/\./, { token: 'delimiter', bracket: '@close', next: '@pop' }]
+		],
+
+		identifier_rest: [
+			[/[^\\.]+/, 'identifier'],
+			[/\./, { token: 'delimiter', bracket: '@close', next: '@pop' }]
+		],
+
+		semver: [
+			{ include: '@whitespace' },
+			[':', 'delimiter'],
+			[/\d*\.\d*\.\d*/, { token: 'number.semver', bracket: '@close', next: '@pop' }]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+		],
+	},
+};

src/monaco.contribution.ts

@@ -15,6 +15,7 @@ import './cpp/cpp.contribution';
 import './csharp/csharp.contribution';
 import './csp/csp.contribution';
 import './css/css.contribution';
+import './dart/dart.contribution';
 import './dockerfile/dockerfile.contribution';
 import './fsharp/fsharp.contribution';
 import './go/go.contribution';
@@ -27,6 +28,7 @@ import './javascript/javascript.contribution';
 import './julia/julia.contribution';
 import './kotlin/kotlin.contribution';
 import './less/less.contribution';
+import './lexon/lexon.contribution';
 import './lua/lua.contribution';
 import './markdown/markdown.contribution';
 import './mips/mips.contribution';

src/python/python.ts

@@ -55,9 +55,18 @@ export const language = <ILanguage>{
 	tokenPostfix: '.python',
 
 	keywords: [
+		// This section is the result of running
+		// `for k in keyword.kwlist: print('  "' + k + '",')` in a Python REPL,
+		// though note that the output from Python 3 is not a strict superset of the
+		// output from Python 2.
+		'False', // promoted to keyword.kwlist in Python 3
+		'None', // promoted to keyword.kwlist in Python 3
+		'True', // promoted to keyword.kwlist in Python 3
 		'and',
 		'as',
 		'assert',
+		'async', // new in Python 3
+		'await', // new in Python 3
 		'break',
 		'class',
 		'continue',
@@ -66,7 +75,7 @@ export const language = <ILanguage>{
 		'elif',
 		'else',
 		'except',
-		'exec',
+		'exec', // Python 2, but not 3.
 		'finally',
 		'for',
 		'from',
@@ -76,14 +85,13 @@ export const language = <ILanguage>{
 		'in',
 		'is',
 		'lambda',
-		'None',
+		'nonlocal', // new in Python 3
 		'not',
 		'or',
 		'pass',
-		'print',
+		'print', // Python 2, but not 3.
 		'raise',
 		'return',
-		'self',
 		'try',
 		'while',
 		'with',
@@ -156,6 +164,7 @@ export const language = <ILanguage>{
 		'repr',
 		'reversed',
 		'round',
+		'self',
 		'set',
 		'setattr',
 		'slice',
@@ -172,9 +181,6 @@ export const language = <ILanguage>{
 		'xrange',
 		'zip',
 
-		'True',
-		'False',
-
 		'__dict__',
 		'__methods__',
 		'__members__',

src/r/r.contribution.ts

@@ -8,7 +8,7 @@ import { registerLanguage } from '../_.contribution';
 
 registerLanguage({
 	id: 'r',
-	extensions: ['.r', '.rhistory', '.rprofile', '.rt'],
+	extensions: ['.r', '.rhistory', '.rmd', '.rprofile', '.rt'],
 	aliases: ['R', 'r'],
 	loader: () => import('./r')
 });

src/scala/scala.contribution.ts

@@ -8,8 +8,8 @@ import { registerLanguage } from '../_.contribution';
 
 registerLanguage({
 	id: 'scala',
-	extensions: ['.scala', '.sbt', '.sc'],
+	extensions: ['.scala', '.sc', '.sbt'],
 	aliases: ['Scala', 'scala', 'SBT', 'Sbt', 'sbt', 'Dotty', 'dotty'],
-	mimetypes: ['text/x-scala', 'text/x-sbt', 'text/x-dotty'],
+	mimetypes: ['text/x-scala-source', 'text/x-scala', 'text/x-sbt', 'text/x-dotty'],
 	loader: () => import('./scala')
 });

src/scala/scala.test.ts

@@ -34,5 +34,655 @@ testTokenization('scala', [
 			{startIndex: 7, type: 'white.scala'},
 			{startIndex: 8, type: 'number.scala'}
 		]
+	}],
+
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: 'white.scala' },
+			{ startIndex: 4, type: 'comment.scala' }
+		]
+	}],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[{
+		line: '/* //*/ a',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: '/almost a comment',
+		tokens: [
+			{ startIndex: 0, type: 'operator.scala' },
+			{ startIndex: 1, type: 'identifier.scala' },
+			{ startIndex: 7, type: 'white.scala' },
+			{ startIndex: 8, type: 'keyword.flow.scala' },
+			{ startIndex: 9, type: 'white.scala' },
+			{ startIndex: 10, type: 'identifier.scala' }
+		]
+	}],
+
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'white.scala' },
+			{ startIndex: 2, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'number.scala' },
+			{ startIndex: 5, type: 'delimiter.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: 'val x: Int = 1; // my comment // is a nice one',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'variable.scala' },
+			{ startIndex: 5, type: 'operator.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'type.scala' },
+			{ startIndex: 10, type: 'white.scala' },
+			{ startIndex: 11, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 12, type: 'white.scala' },
+			{ startIndex: 13, type: 'number.scala' },
+			{ startIndex: 14, type: 'delimiter.scala' },
+			{ startIndex: 15, type: 'white.scala' },
+			{ startIndex: 16, type: 'comment.scala' }
+		]
+	}],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: 'val x: Int = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'variable.scala' },
+			{ startIndex: 5, type: 'operator.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'type.scala' },
+			{ startIndex: 10, type: 'white.scala' },
+			{ startIndex: 11, type: 'operator.scala' },
+			{ startIndex: 12, type: 'white.scala' },
+			{ startIndex: 13, type: 'comment.scala' },
+			{ startIndex: 35, type: 'white.scala' },
+			{ startIndex: 36, type: 'number.scala' },
+			{ startIndex: 37, type: 'delimiter.scala' }
+		]
+	}],
+
+	[{
+		line: 'val x: Int = /* a simple comment */ 1; */',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'variable.scala' },
+			{ startIndex: 5, type: 'operator.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'type.scala' },
+			{ startIndex: 10, type: 'white.scala' },
+			{ startIndex: 11, type: 'operator.scala' },
+			{ startIndex: 12, type: 'white.scala' },
+			{ startIndex: 13, type: 'comment.scala' },
+			{ startIndex: 35, type: 'white.scala' },
+			{ startIndex: 36, type: 'number.scala' },
+			{ startIndex: 37, type: 'delimiter.scala' },
+			{ startIndex: 38, type: 'white.scala' },
+			{ startIndex: 39, type: 'operator.scala' },
+		]
+	}],
+
+	[{
+		line: 'x = /**/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.scala' },
+			{ startIndex: 1, type: 'white.scala' },
+			{ startIndex: 2, type: 'operator.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'comment.scala' },
+			{ startIndex: 8, type: 'delimiter.scala' }
+		]
+	}],
+
+	[{
+		line: 'x = /*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.scala' },
+			{ startIndex: 1, type: 'white.scala' },
+			{ startIndex: 2, type: 'operator.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'comment.scala' }
+		]
+	}],
+
+	// Comments - range comment, multiple lines
+	[{
+		line: '/* start of multiline comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}, {
+		line: 'a comment between without a star',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}, {
+		line: 'end of multiline comment*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	[{
+		line: 'val x: Int = /* start a comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'variable.scala' },
+			{ startIndex: 5, type: 'operator.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'type.scala' },
+			{ startIndex: 10, type: 'white.scala' },
+			{ startIndex: 11, type: 'operator.scala' },
+			{ startIndex: 12, type: 'white.scala' },
+			{ startIndex: 13, type: 'comment.scala' },
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' },
+		]
+	}, {
+		line: 'and end it */ 2;',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' },
+			{ startIndex: 13, type: 'white.scala' },
+			{ startIndex: 14, type: 'number.scala' },
+			{ startIndex: 15, type: 'delimiter.scala' }
+		]
+	}],
+
+	// Scala Doc, multiple lines
+	[{
+		line: '/** start of Scala Doc',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}, {
+		line: 'a comment between without a star',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}, {
+		line: 'end of multiline comment*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.scala' }
+		]
+	}],
+
+	// Keywords
+	[{
+		line: 'package test; object Program { def main(args: Array[String]): Unit = {} }',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 7, type: 'white.scala' },
+			{ startIndex: 8, type: 'identifier.scala' },
+			{ startIndex: 12, type: 'delimiter.scala' },
+			{ startIndex: 13, type: 'white.scala' },
+			{ startIndex: 14, type: 'keyword.scala' },
+			{ startIndex: 20, type: 'white.scala' },
+			{ startIndex: 21, type: 'type.scala' },
+			{ startIndex: 28, type: 'white.scala' },
+			{ startIndex: 29, type: 'delimiter.curly.scala' },
+			{ startIndex: 30, type: 'white.scala' },
+			{ startIndex: 31, type: 'keyword.scala' },
+			{ startIndex: 34, type: 'white.scala' },
+			{ startIndex: 35, type: 'identifier.scala' },
+			{ startIndex: 39, type: 'delimiter.parenthesis.scala' },
+			{ startIndex: 40, type: 'variable.scala' },
+			{ startIndex: 44, type: 'operator.scala' },
+			{ startIndex: 45, type: 'white.scala' },
+			{ startIndex: 46, type: 'type.scala' },
+			{ startIndex: 51, type: 'operator.square.scala' },
+			{ startIndex: 52, type: 'type.scala' },
+			{ startIndex: 58, type: 'operator.square.scala' },
+			{ startIndex: 59, type: 'delimiter.parenthesis.scala' },
+			{ startIndex: 60, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 61, type: 'white.scala' },
+			{ startIndex: 62, type: 'type.scala' },
+			{ startIndex: 66, type: 'white.scala' },
+			{ startIndex: 67, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 68, type: 'white.scala' },
+			{ startIndex: 69, type: 'delimiter.curly.scala' },
+			{ startIndex: 71, type: 'white.scala' },
+			{ startIndex: 72, type: 'delimiter.curly.scala' }
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '0.10',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'identifier.scala' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.scala' }
+		]
+	}],
+
+	[{
+		line: '0x5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.scala' }
+		]
+	}],
+
+	[{
+		line: '10e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '10f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5e-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5E3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5E-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5F',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23.5d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '1.72E3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '1.72E3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '1.72E-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '1.72e3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '1.72e3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '1.72e-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '23L',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '23l',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '5_______2',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '3_.1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'keyword.scala' }, // TODO
+			{ startIndex: 2, type: 'delimiter.scala' },
+			{ startIndex: 3, type: 'number.float.scala' }
+		]
+	}],
+
+	[{
+		line: '3._1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'operator.scala' },
+			{ startIndex: 2, type: 'keyword.flow.scala' } // TODO
+		]
+	}],
+
+	[{
+		line: '999_99_9999_L',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 11, type: 'identifier.scala' }
+		]
+	}],
+
+	[{
+		line: '52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 2, type: 'keyword.scala' } // TODO
+		]
+	}],
+
+	[{
+		line: '0_x52',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'identifier.scala' }
+		]
+	}],
+
+	[{
+		line: '0x_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'identifier.scala' }
+		]
+	}],
+
+	[{
+		line: '0x52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.scala' },
+			{ startIndex: 4, type: 'keyword.scala' } // TODO
+		]
+	}],
+
+	[{
+		line: '23.5L',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.scala' },
+			{ startIndex: 4, type: 'type.scala' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'operator.scala' },
+			{ startIndex: 2, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 3, type: 'operator.scala' },
+			{ startIndex: 4, type: 'number.scala' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.scala' },
+			{ startIndex: 1, type: 'white.scala' },
+			{ startIndex: 2, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'number.scala' }
+		]
+	}],
+
+	// single line Strings
+	[{
+		line: 'val s: String = "I\'m a Scala String";',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'variable.scala' },
+			{ startIndex: 5, type: 'operator.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'type.scala' },
+			{ startIndex: 13, type: 'white.scala' },
+			{ startIndex: 14, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 15, type: 'white.scala' },
+			{ startIndex: 16, type: 'string.quote.scala' },
+			{ startIndex: 17, type: 'string.scala' },
+			{ startIndex: 35, type: 'string.quote.scala' },
+			{ startIndex: 36, type: 'delimiter.scala' }
+		]
+	}],
+
+	[{
+		line: 'val s: String = "concatenated" + " String" ;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.scala' },
+			{ startIndex: 3, type: 'white.scala' },
+			{ startIndex: 4, type: 'variable.scala' },
+			{ startIndex: 5, type: 'operator.scala' },
+			{ startIndex: 6, type: 'white.scala' },
+			{ startIndex: 7, type: 'type.scala' },
+			{ startIndex: 13, type: 'white.scala' },
+			{ startIndex: 14, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 15, type: 'white.scala' },
+			{ startIndex: 16, type: 'string.quote.scala' },
+			{ startIndex: 17, type: 'string.scala' },
+			{ startIndex: 29, type: 'string.quote.scala' },
+			{ startIndex: 30, type: 'white.scala' },
+			{ startIndex: 31, type: 'keyword.flow.scala' }, // TODO
+			{ startIndex: 32, type: 'white.scala' },
+			{ startIndex: 33, type: 'string.quote.scala' },
+			{ startIndex: 34, type: 'string.scala' },
+			{ startIndex: 41, type: 'string.quote.scala' },
+			{ startIndex: 42, type: 'white.scala' },
+			{ startIndex: 43, type: 'delimiter.scala' }
+		]
+	}],
+
+	[{
+		line: '"quote in a string"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.scala' },
+			{ startIndex: 1, type: 'string.scala' },
+			{ startIndex: 18, type: 'string.quote.scala' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.scala' },
+			{ startIndex: 1, type: 'string.scala' },
+			{ startIndex: 10, type: 'string.escape.scala' },
+			{ startIndex: 12, type: 'string.scala' },
+			{ startIndex: 18, type: 'string.escape.scala' },
+			{ startIndex: 20, type: 'string.scala' },
+			{ startIndex: 28, type: 'string.quote.scala' },
+		]
+	}],
+
+	[{
+		line: '"\\"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.scala' },
+			{ startIndex: 1, type: 'string.escape.scala' }
+		]
+	}],
+
+	// Annotations
+	[{
+		line: '@',
+		tokens: [
+			{ startIndex: 0, type: 'operator.scala' }
+		]
+	}],
+
+	[{
+		line: '@uncheckedStable',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.scala' }
+		]
+	}],
+
+	[{
+		line: '@silent("deprecated")',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.scala' },
+			{ startIndex: 7, type: 'delimiter.parenthesis.scala' },
+			{ startIndex: 8, type: 'string.quote.scala' },
+			{ startIndex: 9, type: 'string.scala' },
+			{ startIndex: 19, type: 'string.quote.scala' },
+			{ startIndex: 20, type: 'delimiter.parenthesis.scala' }
+		]
+	}],
+
+	[{
+		line: '@AnnotationWithKeywordAfter private',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.scala' },
+			{ startIndex: 27, type: 'white.scala' },
+			{ startIndex: 28, type: 'keyword.modifier.scala' }
+		]
 	}]
 ]);

src/scala/scala.ts

@@ -99,6 +99,8 @@ export const language = <ILanguage>{
 
 	// we include these common regular expressions
 	symbols: /[=><!~?:&|+\-*\/^\\%@#]+/,
+	digits: /\d+(_+\d+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
 
 	// C# style strings
 	escapes: /\\(?:[btnfr\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
@@ -122,27 +124,29 @@ export const language = <ILanguage>{
 			[/"/, {token: 'string.quote', bracket: '@open', next: '@string'}],
 
 			// numbers
-			[/[+\-]?(?:\d[_\d])*\.\d+[dDfFlL]?([eE][\-+]?\d+)?/, 'number.float', '@allowMethod'],
-			[/0[xX][0-9a-fA-F]+/, 'number.hex', '@allowMethod'],
-			[/[+\-]?\d[_\d]*[dDfFlL]?/, 'number', '@allowMethod'],
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float', '@allowMethod'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float', '@allowMethod'],
+			[/0[xX](@hexdigits)[Ll]?/, 'number.hex', '@allowMethod'],
+			[/(@digits)[fFdD]/, 'number.float', '@allowMethod'],
+			[/(@digits)[lL]?/, 'number', '@allowMethod'],
 
 			[/\b_\*/, 'key'],
 			[/\b(_)\b/, 'keyword', '@allowMethod'],
 
 			// identifiers and keywords
 			[/\bimport\b/, 'keyword', '@import'],
-			[/\b(case)([ \t]+)(class)\b/, ['tag.id.pug', 'white', 'keyword']],
+			[/\b(case)([ \t]+)(class)\b/, ['keyword.modifier', 'white', 'keyword']],
 			[/\bcase\b/, 'keyword', '@case'],
 			[/\bva[lr]\b/, 'keyword', '@vardef'],
-			[/\b(def[ \t]+)((?:unary_)?@symbols|@name(?:_=)|@name)/, ['keyword', 'keyword.flow']],
+			[/\b(def)([ \t]+)((?:unary_)?@symbols|@name(?:_=)|@name)/, ['keyword', 'white', 'identifier']],
 			[/@name(?=[ \t]*:(?!:))/, 'variable'],
 			[/(\.)(@name|@symbols)/, ['operator', {token: 'keyword.flow', next: '@allowMethod'}]],
 			[/([{(])(\s*)(@name(?=\s*=>))/, ['@brackets', 'white', 'variable']],
 			[/@name/, {cases: {
 				'@keywords': 'keyword',
 				'@softKeywords': 'keyword',
-				'@modifiers': 'tag.id.pug',
-				'@softModifiers': 'tag.id.pug',
+				'@modifiers': 'keyword.modifier',
+				'@softModifiers': 'keyword.modifier',
 				'@constants': {token: 'constant', next: '@allowMethod'},
 				'@default': {token: 'identifier', next: '@allowMethod'}
 			}}],
@@ -157,9 +161,9 @@ export const language = <ILanguage>{
 			// delimiters and operators
 			[/[{(]/, '@brackets'],
 			[/[})]/, '@brackets', '@allowMethod'],
-			[/\[/, 'operator.scss'],
-			[/](?!\s*(?:va[rl]|def|type)\b)/, 'operator.scss', '@allowMethod'],
-			[/]/, 'operator.scss'],
+			[/\[/, 'operator.square'],
+			[/](?!\s*(?:va[rl]|def|type)\b)/, 'operator.square', '@allowMethod'],
+			[/]/, 'operator.square'],
 			[/([=-]>|<-|>:|<:|:>|<%)(?=[\s\w()[\]{},\."'`])/, 'keyword'],
 			[/@symbols/, 'operator'],
 
@@ -183,7 +187,7 @@ export const language = <ILanguage>{
 			[/\/\*/, 'comment', '@comment'],
 			[/@name|@type/, 'type'],
 			[/[(){}]/, '@brackets'],
-			[/[[\]]/, 'operator.scss'],
+			[/[[\]]/, 'operator.square'],
 			[/[\.,]/, 'delimiter'],
 		],
 
@@ -253,9 +257,9 @@ export const language = <ILanguage>{
 			[/(\$)([a-z_]\w*)/, ['operator', 'identifier']],
 			[/\$\{/, 'operator', '@interp'],
 			[/%%/, 'string'],
-			[/(%)([\-#+ 0,(])(\d+|\.\d+|\d+\.\d+)(@fstring_conv)/, ['metatag', 'tag.id.pug', 'number', 'metatag']],
+			[/(%)([\-#+ 0,(])(\d+|\.\d+|\d+\.\d+)(@fstring_conv)/, ['metatag', 'keyword.modifier', 'number', 'metatag']],
 			[/(%)(\d+|\.\d+|\d+\.\d+)(@fstring_conv)/, ['metatag', 'number', 'metatag']],
-			[/(%)([\-#+ 0,(])(@fstring_conv)/, ['metatag', 'tag.id.pug', 'metatag']],
+			[/(%)([\-#+ 0,(])(@fstring_conv)/, ['metatag', 'keyword.modifier', 'metatag']],
 			[/(%)(@fstring_conv)/, ['metatag', 'metatag']],
 			[/./, 'string']
 		],
@@ -268,9 +272,9 @@ export const language = <ILanguage>{
 			[/(\$)([a-z_]\w*)/, ['operator', 'identifier']],
 			[/\$\{/, 'operator', '@interp'],
 			[/%%/, 'string'],
-			[/(%)([\-#+ 0,(])(\d+|\.\d+|\d+\.\d+)(@fstring_conv)/, ['metatag', 'tag.id.pug', 'number', 'metatag']],
+			[/(%)([\-#+ 0,(])(\d+|\.\d+|\d+\.\d+)(@fstring_conv)/, ['metatag', 'keyword.modifier', 'number', 'metatag']],
 			[/(%)(\d+|\.\d+|\d+\.\d+)(@fstring_conv)/, ['metatag', 'number', 'metatag']],
-			[/(%)([\-#+ 0,(])(@fstring_conv)/, ['metatag', 'tag.id.pug', 'metatag']],
+			[/(%)([\-#+ 0,(])(@fstring_conv)/, ['metatag', 'keyword.modifier', 'metatag']],
 			[/(%)(@fstring_conv)/, ['metatag', 'metatag']],
 			[/./, 'string']
 		],

src/test/assert.d.ts

@@ -1,44 +0,0 @@
-declare module "assert" {
-    function internal (value: any, message?: string): void;
-    namespace internal {
-        export class AssertionError implements Error {
-            name: string;
-            message: string;
-            actual: any;
-            expected: any;
-            operator: string;
-            generatedMessage: boolean;
-
-            constructor(options?: {message?: string; actual?: any; expected?: any;
-                                  operator?: string; stackStartFunction?: Function});
-        }
-
-        export function fail(actual?: any, expected?: any, message?: string, operator?: string): void;
-        export function ok(value: any, message?: string): void;
-        export function equal(actual: any, expected: any, message?: string): void;
-        export function notEqual(actual: any, expected: any, message?: string): void;
-        export function deepEqual(actual: any, expected: any, message?: string): void;
-        export function notDeepEqual(acutal: any, expected: any, message?: string): void;
-        export function strictEqual(actual: any, expected: any, message?: string): void;
-        export function notStrictEqual(actual: any, expected: any, message?: string): void;
-        export function deepStrictEqual(actual: any, expected: any, message?: string): void;
-        export function notDeepStrictEqual(actual: any, expected: any, message?: string): void;
-        export var throws: {
-            (block: Function, message?: string): void;
-            (block: Function, error: Function, message?: string): void;
-            (block: Function, error: RegExp, message?: string): void;
-            (block: Function, error: (err: any) => boolean, message?: string): void;
-        };
-
-        export var doesNotThrow: {
-            (block: Function, message?: string): void;
-            (block: Function, error: Function, message?: string): void;
-            (block: Function, error: RegExp, message?: string): void;
-            (block: Function, error: (err: any) => boolean, message?: string): void;
-        };
-
-        export function ifError(value: any): void;
-    }
-
-    export = internal;
-}

src/test/testRunner.ts

@@ -5,7 +5,7 @@
 
 import '../monaco.contribution';
 import {loadLanguage} from '../_.contribution';
-import * as assert from 'assert';
+import * as test from 'tape';
 
 // Allow for running under nodejs/requirejs in tests
 const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
@@ -28,24 +28,23 @@ export function testTokenization(_language:string|string[], tests:ITestItem[][])
 		languages = _language;
 	}
 	let mainLanguage = languages[0];
-	suite(mainLanguage + ' tokenization', () => {
-		test('', (done) => {
-			Promise.all(languages.map(l => loadLanguage(l))).then(() => {
-				// clean stack
-				setTimeout(() => {
-					runTests(mainLanguage, tests);
-					done();
-				});
-			}).then(null, done);
-		});
+
+	test(mainLanguage + ' tokenization', (t: test.Test) => {
+		Promise.all(languages.map(l => loadLanguage(l))).then(() => {
+			// clean stack
+			setTimeout(() => {
+				runTests(t, mainLanguage, tests);
+				t.end();
+			});
+		}).then(null, () => t.end());
 	});
 }
 
-function runTests(languageId:string, tests:ITestItem[][]): void {
-	tests.forEach((test) => runTest(languageId, test));
+function runTests(t: test.Test, languageId:string, tests:ITestItem[][]): void {
+	tests.forEach((test) => runTest(t, languageId, test));
 }
 
-function runTest(languageId:string, test:ITestItem[]): void {
+function runTest(t: test.Test, languageId:string, test:ITestItem[]): void {
 
 	let text = test.map(t => t.line).join('\n');
 	let actualTokens = _monaco.editor.tokenize(text, languageId);
@@ -61,5 +60,5 @@ function runTest(languageId:string, test:ITestItem[]): void {
 		};
 	});
 
-	assert.deepEqual(actual, test);
+	t.deepEqual(actual, test);
 }

src/typescript/typescript.ts

@@ -73,19 +73,17 @@ export const language = {
 	tokenPostfix: '.ts',
 
 	keywords: [
-		'abstract', 'as', 'break', 'case', 'catch', 'class', 'continue', 'const',
-		'constructor', 'debugger', 'declare', 'default', 'delete', 'do', 'else',
-		'enum', 'export', 'extends', 'false', 'finally', 'for', 'from', 'function',
-		'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof', 'interface',
-		'is', 'keyof', 'let', 'module', 'namespace', 'never', 'new', 'null', 'package',
-		'private', 'protected', 'public', 'readonly', 'require', 'global', 'return',
-		'set', 'static', 'super', 'switch', 'symbol', 'this', 'throw', 'true', 'try',
-		'type', 'typeof', 'unique', 'var', 'void', 'while', 'with', 'yield', 'async',
-		'await', 'of'
-	],
-
-	typeKeywords: [
-		'any', 'boolean', 'number', 'object', 'string', 'undefined'
+		// Should match the keys of textToKeywordObj in
+		// https://github.com/microsoft/TypeScript/blob/master/src/compiler/scanner.ts
+		'abstract', 'any', 'as', 'asserts', 'bigint', 'boolean', 'break', 'case', 'catch',
+		'class', 'continue', 'const', 'constructor', 'debugger', 'declare', 'default',
+		'delete', 'do', 'else', 'enum', 'export', 'extends', 'false', 'finally', 'for',
+		'from', 'function', 'get', 'if', 'implements', 'import', 'in', 'infer',
+		'instanceof', 'interface', 'is', 'keyof', 'let', 'module', 'namespace', 'never',
+		'new', 'null', 'number', 'object', 'package', 'private', 'protected', 'public',
+		'readonly', 'require', 'global', 'return', 'set', 'static', 'string', 'super',
+		'switch', 'symbol', 'this', 'throw', 'true', 'try', 'type', 'typeof', 'undefined',
+		'unique', 'unknown', 'var', 'void', 'while', 'with', 'yield', 'async', 'await', 'of'
 	],
 
 	operators: [
@@ -118,7 +116,6 @@ export const language = {
 			// identifiers and keywords
 			[/[a-z_$][\w$]*/, {
 				cases: {
-					'@typeKeywords': 'keyword',
 					'@keywords': 'keyword',
 					'@default': 'identifier'
 				}

src/vb/vb.test.ts

@@ -289,18 +289,95 @@ testTokenization('vb', [
 			{ startIndex: 8, type: '' },
 			{ startIndex: 9, type: 'delimiter.vb' },
 			{ startIndex: 10, type: '' },
-			{ startIndex: 11, type: 'string.vb' }
+			{ startIndex: 11, type: 'string.quote.vb' },
+			{ startIndex: 12, type: 'string.vb' },
+			{ startIndex: 18, type: 'string.quote.vb' }
 		]
 	}],
 
 	[{
 		line: '"use strict";',
 		tokens: [
-			{ startIndex: 0, type: 'string.vb' },
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 11, type: 'string.quote.vb' },
 			{ startIndex: 12, type: 'delimiter.vb' }
 		]
 	}],
 
+	[{
+		line: '"a""b"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 2, type: 'string.escape.vb' },
+			{ startIndex: 4, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '"a““b"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 2, type: 'string.escape.vb' },
+			{ startIndex: 4, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '"a””b"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 2, type: 'string.escape.vb' },
+			{ startIndex: 4, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
+		]
+	}],
+
+	[{
+		line: '"mixed quotes 1“',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '"mixed quotes 2”',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '“mixed quotes 3"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '“mixed quotes 4”',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '”mixed quotes 5"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '”mixed quotes 6“',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}],
+
 	// Tags
 	[{
 		line: 'Public Sub ToString()',
@@ -445,12 +522,14 @@ testTokenization('vb', [
 			{ startIndex: 5, type: '' },
 			{ startIndex: 6, type: 'delimiter.vb' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'string.vb' }
+			{ startIndex: 8, type: 'string.quote.vb' },
+			{ startIndex: 9, type: 'string.vb' }
 		]
 	}, {
 		line: 'world"',
 		tokens: [
 			{ startIndex: 0, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
 		]
 	}],
 

src/vb/vb.ts

@@ -124,7 +124,6 @@ export const language = <ILanguage>{
 
 	// we include these common regular expressions
 	symbols: /[=><!~?;\.,:&|+\-*\/\^%]+/,
-	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
 	integersuffix: /U?[DI%L&S@]?/,
 	floatsuffix: /[R#F!]?/,
 
@@ -169,7 +168,7 @@ export const language = <ILanguage>{
 			[/@symbols/, 'delimiter'],
 
 			// strings
-			[/"/, 'string', '@string'],
+			[/["\u201c\u201d]/, { token: 'string.quote', next: '@string' }],
 
 		],
 
@@ -179,10 +178,9 @@ export const language = <ILanguage>{
 		],
 
 		string: [
-			[/[^\\"]+/, 'string'],
-			[/@escapes/, 'string.escape'],
-			[/\\./, 'string.escape.invalid'],
-			[/"C?/, 'string', '@pop']
+			[/[^"\u201c\u201d]+/, 'string'],
+			[/["\u201c\u201d]{2}/, 'string.escape'],
+			[/["\u201c\u201d]C?/, { token: 'string.quote', next: '@pop' }]
 		],
 	},
 };

test/all.js

@@ -27,7 +27,7 @@ requirejs(['./test/setup'], function () {
 			return;
 		}
 		requirejs(files.map(f => f.replace(/\.js$/, '')), function () {
-			run(); // We can launch the tests!
+			// We can launch the tests!
 		}, function (err) {
 			console.log(err);
 		})

test/mocha.opts

@@ -1,3 +0,0 @@
---delay
---ui tdd
-test/all.js