Eliminate htmlTokenTypes

src/bat.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: 'REM'
 	},
@@ -30,7 +30,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	ignoreCase: true,
 	tokenPostfix: '.bat',

src/coffee.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\$\-\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
 	comments: {
 		blockComment: ['###', '###'],
@@ -35,7 +35,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	ignoreCase: true,
 	tokenPostfix: '.coffee',

src/cpp.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['/*', '*/'],
@@ -34,7 +34,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.cpp',
 

src/csharp.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
 	comments: {
 		lineComment: '//',
@@ -36,7 +36,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.cs',
 

src/css.ts

@@ -8,7 +8,7 @@
 import LanguageConfiguration = monaco.languages.LanguageConfiguration;
 import IMonarchLanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: LanguageConfiguration = {
+export const conf: LanguageConfiguration = {
 	wordPattern: /(#?-?\d*\.\d\w*%?)|((::|[@#.!:])?[\w-?]+%?)|::|[@#.!:]/g,
 
 	comments: {
@@ -38,13 +38,7 @@ export var conf: LanguageConfiguration = {
 	]
 };
 
-const TOKEN_SELECTOR = 'tag';
-const TOKEN_SELECTOR_TAG = 'tag';
-const TOKEN_PROPERTY = 'attribute.name';
-const TOKEN_VALUE = 'attribute.value';
-const TOKEN_AT_KEYWORD = 'keyword';
-
-export var language = <IMonarchLanguage>{
+export const language = <IMonarchLanguage>{
 	defaultToken: '',
 	tokenPostfix: '.css',
 
@@ -67,13 +61,13 @@ export var language = <IMonarchLanguage>{
 			{ include: '@comments' },
 			{ include: '@import' },
 			{ include: '@strings' },
-			['[@](keyframes|-webkit-keyframes|-moz-keyframes|-o-keyframes)', { token: TOKEN_AT_KEYWORD, next: '@keyframedeclaration' }],
-			['[@](page|content|font-face|-moz-document)', { token: TOKEN_AT_KEYWORD }],
-			['[@](charset|namespace)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }],
+			['[@](keyframes|-webkit-keyframes|-moz-keyframes|-o-keyframes)', { token: 'keyword', next: '@keyframedeclaration' }],
+			['[@](page|content|font-face|-moz-document)', { token: 'keyword' }],
+			['[@](charset|namespace)', { token: 'keyword', next: '@declarationbody' }],
 			['(url-prefix)(\\()', ['attribute.value', { token: 'delimiter.parenthesis', next: '@urldeclaration' }]],
 			['(url)(\\()', ['attribute.value', { token: 'delimiter.parenthesis', next: '@urldeclaration' }]],
 			{ include: '@selectorname' },
-			['[\\*]', TOKEN_SELECTOR_TAG], // selector symbols
+			['[\\*]', 'tag'], // selector symbols
 			['[>\\+,]', 'delimiter'], // selector operators
 			['\\[', { token: 'delimiter.bracket', next: '@selectorattribute' }],
 			['{', { token: 'delimiter.bracket', next: '@selectorbody' }]
@@ -81,12 +75,12 @@ export var language = <IMonarchLanguage>{
 
 		selectorbody: [
 			{ include: '@comments' },
-			['[*_]?@identifier@ws:(?=(\\s|\\d|[^{;}]*[;}]))', TOKEN_PROPERTY, '@rulevalue'], // rule definition: to distinguish from a nested selector check for whitespace, number or a semicolon
+			['[*_]?@identifier@ws:(?=(\\s|\\d|[^{;}]*[;}]))', 'attribute.name', '@rulevalue'], // rule definition: to distinguish from a nested selector check for whitespace, number or a semicolon
 			['}', { token: 'delimiter.bracket', next: '@pop' }]
 		],
 
 		selectorname: [
-			['(\\.|#(?=[^{])|%|(@identifier)|:)+', TOKEN_SELECTOR], // selector (.foo, div, ...)
+			['(\\.|#(?=[^{])|%|(@identifier)|:)+', 'tag'], // selector (.foo, div, ...)
 		],
 
 		selectorattribute: [
@@ -115,11 +109,11 @@ export var language = <IMonarchLanguage>{
 		],
 
 		warndebug: [
-			['[@](warn|debug)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }]
+			['[@](warn|debug)', { token: 'keyword', next: '@declarationbody' }]
 		],
 
 		import: [
-			['[@](import)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }]
+			['[@](import)', { token: 'keyword', next: '@declarationbody' }]
 		],
 
 		urldeclaration: [
@@ -151,7 +145,7 @@ export var language = <IMonarchLanguage>{
 		],
 
 		name: [
-			['@identifier', TOKEN_VALUE]
+			['@identifier', 'attribute.value']
 		],
 
 		numbers: [
@@ -179,7 +173,7 @@ export var language = <IMonarchLanguage>{
 		],
 
 		functionarguments: [
-			['\\$@identifier@ws:', TOKEN_PROPERTY],
+			['\\$@identifier@ws:', 'attribute.name'],
 			['[,]', 'delimiter'],
 			{ include: '@term' },
 			['\\)', { token: 'attribute.value', next: '@pop' }],

src/dockerfile.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	brackets: [
 		['{', '}'],
 		['[', ']'],
@@ -30,7 +30,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.dockerfile',
 

src/fsharp.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['(*', '*)'],
@@ -33,7 +33,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.fs',
 

src/go.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['/*', '*/'],
@@ -36,7 +36,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 
 	defaultToken: '',
 	tokenPostfix: '.go',

src/handlebars.ts

@@ -9,11 +9,11 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
 // Allow for running under nodejs/requirejs in tests
-var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
 
 const EMPTY_ELEMENTS: string[] = ['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr'];
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
 
 	comments: {
@@ -55,17 +55,7 @@ export var conf: IRichLanguageConfiguration = {
 	],
 }
 
-export const htmlTokenTypes = {
-	DELIM_START: 'delimiter.html',
-	DELIM_END: 'delimiter.html',
-	DELIM_COMMENT: 'comment.html',
-	COMMENT: 'comment.content.html',
-	getTag: (name: string) => {
-		return 'tag.html';
-	}
-};
-
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '',
 	// ignoreCase: true,
@@ -76,13 +66,13 @@ export var language = <ILanguage>{
 			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.root' }],
 			[/<!DOCTYPE/, 'metatag.html', '@doctype'],
 			[/<!--/, 'comment.html', '@comment'],
-			[/(<)(\w+)(\/>)/, [htmlTokenTypes.DELIM_START, 'tag.html', htmlTokenTypes.DELIM_END]],
-			[/(<)(script)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@script' }]],
-			[/(<)(style)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@style' }]],
-			[/(<)([:\w]+)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@otherTag' }]],
-			[/(<\/)(\w+)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@otherTag' }]],
-			[/</, htmlTokenTypes.DELIM_START],
-			[/\{/, htmlTokenTypes.DELIM_START],
+			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
+			[/(<)(script)/, ['delimiter.html', { token: 'tag.html', next: '@script' }]],
+			[/(<)(style)/, ['delimiter.html', { token: 'tag.html', next: '@style' }]],
+			[/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/</, 'delimiter.html'],
+			[/\{/, 'delimiter.html'],
 			[/[^<{]+/] // text
 		],
 
@@ -101,7 +91,7 @@ export var language = <ILanguage>{
 
 		otherTag: [
 			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.otherTag' }],
-			[/\/?>/, htmlTokenTypes.DELIM_END, '@pop'],
+			[/\/?>/, 'delimiter.html', '@pop'],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -119,9 +109,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.text/javascript', nextEmbedded: 'text/javascript' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.text/javascript', nextEmbedded: 'text/javascript' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(script\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(script\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
 		],
 
 		// After <script ... type
@@ -144,7 +134,7 @@ export var language = <ILanguage>{
 		// After <script ... type = $S2
 		scriptWithCustomType: [
 			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.scriptWithCustomType.$S2' }],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -171,9 +161,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.text/css', nextEmbedded: 'text/css' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.text/css', nextEmbedded: 'text/css' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(style\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(style\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
 		],
 
 		// After <style ... type
@@ -196,7 +186,7 @@ export var language = <ILanguage>{
 		// After <style ... type = $S2
 		styleWithCustomType: [
 			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.styleWithCustomType.$S2' }],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],

src/html.ts

@@ -9,11 +9,11 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
 // Allow for running under nodejs/requirejs in tests
-var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
 
 const EMPTY_ELEMENTS: string[] = ['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr'];
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
 
 	comments: {
@@ -57,16 +57,7 @@ export var conf: IRichLanguageConfiguration = {
 	],
 };
 
-export const htmlTokenTypes = {
-	DELIM_START: 'delimiter',
-	DELIM_END: 'delimiter',
-	DELIM_COMMENT: 'comment',
-	getTag: (name: string) => {
-		return 'tag';
-	}
-};
-
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.html',
 	ignoreCase: true,
@@ -76,12 +67,12 @@ export var language = <ILanguage>{
 		root: [
 			[/<!DOCTYPE/, 'metatag', '@doctype'],
 			[/<!--/, 'comment', '@comment'],
-			[/(<)((?:[\w\-]+:)?[\w\-]+)(\s*)(\/>)/, [htmlTokenTypes.DELIM_START, 'tag', '', htmlTokenTypes.DELIM_END]],
-			[/(<)(script)/, [htmlTokenTypes.DELIM_START, { token: 'tag', next: '@script' }]],
-			[/(<)(style)/, [htmlTokenTypes.DELIM_START, { token: 'tag', next: '@style' }]],
-			[/(<)((?:[\w\-]+:)?[\w\-]+)/, [htmlTokenTypes.DELIM_START, { token: 'tag', next: '@otherTag' }]],
-			[/(<\/)((?:[\w\-]+:)?[\w\-]+)/, [htmlTokenTypes.DELIM_START, { token: 'tag', next: '@otherTag' }]],
-			[/</, htmlTokenTypes.DELIM_START],
+			[/(<)((?:[\w\-]+:)?[\w\-]+)(\s*)(\/>)/, ['delimiter', 'tag', '', 'delimiter']],
+			[/(<)(script)/, ['delimiter', { token: 'tag', next: '@script' }]],
+			[/(<)(style)/, ['delimiter', { token: 'tag', next: '@style' }]],
+			[/(<)((?:[\w\-]+:)?[\w\-]+)/, ['delimiter', { token: 'tag', next: '@otherTag' }]],
+			[/(<\/)((?:[\w\-]+:)?[\w\-]+)/, ['delimiter', { token: 'tag', next: '@otherTag' }]],
+			[/</, 'delimiter'],
 			[/[^<]+/], // text
 		],
 
@@ -97,7 +88,7 @@ export var language = <ILanguage>{
 		],
 
 		otherTag: [
-			[/\/?>/, htmlTokenTypes.DELIM_END, '@pop'],
+			[/\/?>/, 'delimiter', '@pop'],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -114,9 +105,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded', nextEmbedded: 'text/javascript' }],
+			[/>/, { token: 'delimiter', next: '@scriptEmbedded', nextEmbedded: 'text/javascript' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(script\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(script\s*)(>)/, ['delimiter', 'tag', { token: 'delimiter', next: '@pop' }]]
 		],
 
 		// After <script ... type
@@ -136,7 +127,7 @@ export var language = <ILanguage>{
 
 		// After <script ... type = $S2
 		scriptWithCustomType: [
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter', next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -162,9 +153,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded', nextEmbedded: 'text/css' }],
+			[/>/, { token: 'delimiter', next: '@styleEmbedded', nextEmbedded: 'text/css' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(style\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(style\s*)(>)/, ['delimiter', 'tag', { token: 'delimiter', next: '@pop' }]]
 		],
 
 		// After <style ... type
@@ -184,7 +175,7 @@ export var language = <ILanguage>{
 
 		// After <style ... type = $S2
 		styleWithCustomType: [
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter', next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],

src/ini.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '#'
 	},
@@ -33,7 +33,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.ini',
 

src/jade.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//'
 	},
@@ -22,7 +22,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.jade',
 

src/java.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	// the default separators except `@$`
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
 	comments: {
@@ -37,7 +37,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.java',
 

src/less.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(#?-?\d*\.\d\w*%?)|([@#!.:]?[\w-?]+%?)|[@#!.]/g,
 	comments: {
 		blockComment: ['/*', '*/'],
@@ -35,13 +35,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-const TOKEN_SELECTOR = 'tag';
-const TOKEN_SELECTOR_TAG = 'tag';
-const TOKEN_PROPERTY = 'attribute.name';
-const TOKEN_VALUE = 'attribute.value';
-const TOKEN_AT_KEYWORD = 'keyword.control.at-rule';
-
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.less',
 
@@ -65,20 +59,20 @@ export var language = <ILanguage>{
 			{ include: '@keyword' },
 			{ include: '@strings' },
 			{ include: '@numbers' },
-			['[*_]?[a-zA-Z\\-\\s]+(?=:.*(;|(\\\\$)))', TOKEN_PROPERTY, '@attribute'],
+			['[*_]?[a-zA-Z\\-\\s]+(?=:.*(;|(\\\\$)))', 'attribute.name', '@attribute'],
 
 			['url(\\-prefix)?\\(', { token: 'tag', next: '@urldeclaration' }],
 
 			['[{}()\\[\\]]', '@brackets'],
 			['[,:;]', 'delimiter'],
 
-			['#@identifierPlus', TOKEN_SELECTOR + '.id'],
-			['&', TOKEN_SELECTOR_TAG],
+			['#@identifierPlus', 'tag.id'],
+			['&', 'tag'],
 
-			['\\.@identifierPlus(?=\\()', TOKEN_SELECTOR + '.class', '@attribute'],
-			['\\.@identifierPlus', TOKEN_SELECTOR + '.class'],
+			['\\.@identifierPlus(?=\\()', 'tag.class', '@attribute'],
+			['\\.@identifierPlus', 'tag.class'],
 
-			['@identifierPlus', TOKEN_SELECTOR_TAG],
+			['@identifierPlus', 'tag'],
 			{ include: '@operators' },
 
 			['@(@identifier(?=[:,\\)]))', 'variable', '@attribute'],
@@ -117,9 +111,9 @@ export var language = <ILanguage>{
 
 			{ include: '@keyword' },
 
-			['[a-zA-Z\\-]+(?=\\()', TOKEN_VALUE, '@attribute'],
+			['[a-zA-Z\\-]+(?=\\()', 'attribute.value', '@attribute'],
 			['>', 'operator', '@pop'],
-			['@identifier', TOKEN_VALUE],
+			['@identifier', 'attribute.value'],
 			{ include: '@operators' },
 			['@(@identifier)', 'variable'],
 
@@ -130,7 +124,7 @@ export var language = <ILanguage>{
 			['[,=:]', 'delimiter'],
 
 			['\\s', ''],
-			['.', TOKEN_VALUE]
+			['.', 'attribute.value']
 		],
 
 		comments: [
@@ -144,12 +138,12 @@ export var language = <ILanguage>{
 		],
 
 		numbers: [
-			['(\\d*\\.)?\\d+([eE][\\-+]?\\d+)?', { token: TOKEN_VALUE + '.number', next: '@units' }],
-			['#[0-9a-fA-F_]+(?!\\w)', TOKEN_VALUE + '.hex']
+			['(\\d*\\.)?\\d+([eE][\\-+]?\\d+)?', { token: 'attribute.value.number', next: '@units' }],
+			['#[0-9a-fA-F_]+(?!\\w)', 'attribute.value.hex']
 		],
 
 		units: [
-			['(em|ex|ch|rem|vmin|vmax|vw|vh|vm|cm|mm|in|px|pt|pc|deg|grad|rad|turn|s|ms|Hz|kHz|%)?', TOKEN_VALUE + '.unit', '@pop']
+			['(em|ex|ch|rem|vmin|vmax|vw|vh|vm|cm|mm|in|px|pt|pc|deg|grad|rad|turn|s|ms|Hz|kHz|%)?', 'attribute.value.unit', '@pop']
 		],
 
 		strings: [

src/lua.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '--',
 		blockComment: ['--[[', ']]'],
@@ -34,7 +34,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.lua',
 

src/markdown.ts

@@ -25,7 +25,7 @@ function getTag(name: string) {
 	return 'tag';
 }
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		blockComment: ['<!--', '-->',]
 	},
@@ -47,7 +47,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.md',
 

src/monaco.contribution.ts

@@ -7,7 +7,7 @@
 declare var require: <T>(moduleId: [string], callback: (module: T) => void, error: (err: any) => void) => void;
 
 // Allow for running under nodejs/requirejs in tests
-var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
 
 interface ILang extends monaco.languages.ILanguageExtensionPoint {
 	module: string;

src/msdax.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['/*', '*/'],
@@ -23,7 +23,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.msdax',
 	ignoreCase: true,

src/objective-c.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['/*', '*/'],
@@ -34,7 +34,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.objective-c',
 

src/php.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
 
 	comments: {
@@ -31,17 +31,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export const htmlTokenTypes = {
-	DELIM_START: 'delimiter.html',
-	DELIM_END: 'delimiter.html',
-	DELIM_COMMENT: 'comment.html',
-	COMMENT: 'comment.content.html',
-	getTag: (name: string) => {
-		return 'tag.html';
-	}
-};
-
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '',
 	// ignoreCase: true,
@@ -52,12 +42,12 @@ export var language = <ILanguage>{
 			[/<\?((php)|=)?/, { token: '@rematch', switchTo: '@phpInSimpleState.root' }],
 			[/<!DOCTYPE/, 'metatag.html', '@doctype'],
 			[/<!--/, 'comment.html', '@comment'],
-			[/(<)(\w+)(\/>)/, [htmlTokenTypes.DELIM_START, 'tag.html', htmlTokenTypes.DELIM_END]],
-			[/(<)(script)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@script' }]],
-			[/(<)(style)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@style' }]],
-			[/(<)([:\w]+)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@otherTag' }]],
-			[/(<\/)(\w+)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@otherTag' }]],
-			[/</, htmlTokenTypes.DELIM_START],
+			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
+			[/(<)(script)/, ['delimiter.html', { token: 'tag.html', next: '@script' }]],
+			[/(<)(style)/, ['delimiter.html', { token: 'tag.html', next: '@style' }]],
+			[/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/</, 'delimiter.html'],
 			[/[^<]+/] // text
 		],
 
@@ -76,7 +66,7 @@ export var language = <ILanguage>{
 
 		otherTag: [
 			[/<\?((php)|=)?/, { token: '@rematch', switchTo: '@phpInSimpleState.otherTag' }],
-			[/\/?>/, htmlTokenTypes.DELIM_END, '@pop'],
+			[/\/?>/, 'delimiter.html', '@pop'],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -94,9 +84,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.text/javascript', nextEmbedded: 'text/javascript' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.text/javascript', nextEmbedded: 'text/javascript' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(script\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(script\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
 		],
 
 		// After <script ... type
@@ -119,7 +109,7 @@ export var language = <ILanguage>{
 		// After <script ... type = $S2
 		scriptWithCustomType: [
 			[/<\?((php)|=)?/, { token: '@rematch', switchTo: '@phpInSimpleState.scriptWithCustomType.$S2' }],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -146,9 +136,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.text/css', nextEmbedded: 'text/css' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.text/css', nextEmbedded: 'text/css' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(style\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(style\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
 		],
 
 		// After <style ... type
@@ -171,7 +161,7 @@ export var language = <ILanguage>{
 		// After <style ... type = $S2
 		styleWithCustomType: [
 			[/<\?((php)|=)?/, { token: '@rematch', switchTo: '@phpInSimpleState.styleWithCustomType.$S2' }],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],

src/postiats.ts

@@ -10,7 +10,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['(*', '*)'],
@@ -24,7 +24,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	tokenPostfix: '.pats',
 
 	// TODO: staload and dynload are followed by a special kind of string literals

src/powershell.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	// the default separators except `$-`
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
 	comments: {
@@ -36,7 +36,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	ignoreCase: true,
 	tokenPostfix: '.ps1',

src/python.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '#',
 		blockComment: ['\'\'\'', '\'\'\''],
@@ -34,7 +34,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.python',
 

src/r.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '#'
 	},
@@ -31,7 +31,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.r',
 

src/razor.ts

@@ -9,11 +9,11 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
 // Allow for running under nodejs/requirejs in tests
-var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
 
 const EMPTY_ELEMENTS: string[] = ['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr'];
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
 
 	comments: {
@@ -54,17 +54,7 @@ export var conf: IRichLanguageConfiguration = {
 	],
 };
 
-export const htmlTokenTypes = {
-	DELIM_START: 'delimiter.html',
-	DELIM_END: 'delimiter.html',
-	DELIM_COMMENT: 'comment.html',
-	COMMENT: 'comment.content.html',
-	getTag: (name: string) => {
-		return 'tag.html';
-	}
-};
-
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '',
 	// ignoreCase: true,
@@ -76,12 +66,12 @@ export var language = <ILanguage>{
 			[/@[^@]/, { token: '@rematch', switchTo: '@razorInSimpleState.root' }],
 			[/<!DOCTYPE/, 'metatag.html', '@doctype'],
 			[/<!--/, 'comment.html', '@comment'],
-			[/(<)(\w+)(\/>)/, [htmlTokenTypes.DELIM_START, 'tag.html', htmlTokenTypes.DELIM_END]],
-			[/(<)(script)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@script' }]],
-			[/(<)(style)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@style' }]],
-			[/(<)([:\w]+)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@otherTag' }]],
-			[/(<\/)(\w+)/, [htmlTokenTypes.DELIM_START, { token: 'tag.html', next: '@otherTag' }]],
-			[/</, htmlTokenTypes.DELIM_START],
+			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
+			[/(<)(script)/, ['delimiter.html', { token: 'tag.html', next: '@script' }]],
+			[/(<)(style)/, ['delimiter.html', { token: 'tag.html', next: '@style' }]],
+			[/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/</, 'delimiter.html'],
 			[/[ \t\r\n]+/], // whitespace
 			[/[^<@]+/], // text
 		],
@@ -101,7 +91,7 @@ export var language = <ILanguage>{
 
 		otherTag: [
 			[/@[^@]/, { token: '@rematch', switchTo: '@razorInSimpleState.otherTag' }],
-			[/\/?>/, htmlTokenTypes.DELIM_END, '@pop'],
+			[/\/?>/, 'delimiter.html', '@pop'],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -119,9 +109,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.text/javascript', nextEmbedded: 'text/javascript' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.text/javascript', nextEmbedded: 'text/javascript' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(script\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(script\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
 		],
 
 		// After <script ... type
@@ -144,7 +134,7 @@ export var language = <ILanguage>{
 		// After <script ... type = $S2
 		scriptWithCustomType: [
 			[/@[^@]/, { token: '@rematch', switchTo: '@razorInSimpleState.scriptWithCustomType.$S2' }],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -171,9 +161,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
 			[/=/, 'delimiter'],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.text/css', nextEmbedded: 'text/css' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.text/css', nextEmbedded: 'text/css' }],
 			[/[ \t\r\n]+/], // whitespace
-			[/(<\/)(style\s*)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', { token: htmlTokenTypes.DELIM_END, next: '@pop' }]]
+			[/(<\/)(style\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
 		],
 
 		// After <style ... type
@@ -196,7 +186,7 @@ export var language = <ILanguage>{
 		// After <style ... type = $S2
 		styleWithCustomType: [
 			[/@[^@]/, { token: '@rematch', switchTo: '@razorInSimpleState.styleWithCustomType.$S2' }],
-			[/>/, { token: htmlTokenTypes.DELIM_END, next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
 			[/"([^"]*)"/, 'attribute.value'],
 			[/'([^']*)'/, 'attribute.value'],
 			[/[\w\-]+/, 'attribute.name'],
@@ -279,9 +269,9 @@ export var language = <ILanguage>{
 			[/'([^']*)'/, 'string.cs'],
 
 			// simple html
-			[/(<)(\w+)(\/>)/, [htmlTokenTypes.DELIM_START, 'tag.html', htmlTokenTypes.DELIM_END]],
-			[/(<)(\w+)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', htmlTokenTypes.DELIM_END]],
-			[/(<\/)(\w+)(>)/, [htmlTokenTypes.DELIM_START, 'tag.html', htmlTokenTypes.DELIM_END]],
+			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
+			[/(<)(\w+)(>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
+			[/(<\/)(\w+)(>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
 
 			// delimiters
 			[/[\+\-\*\%\&\|\^\~\!\=\<\>\/\?\;\:\.\,]/, 'delimiter.cs'],

src/ruby.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '#',
 		blockComment: ['=begin', '=end'],
@@ -83,7 +83,7 @@ export var conf: IRichLanguageConfiguration = {
  * regular expression can also contain comments.
  */
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	tokenPostfix: '.ruby',
 
 	keywords: [

src/scss.ts

@@ -8,7 +8,7 @@
 import LanguageConfiguration = monaco.languages.LanguageConfiguration;
 import IMonarchLanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: LanguageConfiguration = {
+export const conf: LanguageConfiguration = {
 	wordPattern: /(#?-?\d*\.\d\w*%?)|([@$#!.:]?[\w-?]+%?)|[@#!.]/g,
 	comments: {
 		blockComment: ['/*', '*/'],
@@ -35,13 +35,7 @@ export var conf: LanguageConfiguration = {
 	]
 };
 
-const TOKEN_SELECTOR = 'tag';
-const TOKEN_SELECTOR_TAG = 'tag';
-const TOKEN_PROPERTY = 'attribute.name';
-const TOKEN_VALUE = 'attribute.value';
-const TOKEN_AT_KEYWORD = 'keyword';
-
-export var language = <IMonarchLanguage>{
+export const language = <IMonarchLanguage>{
 	defaultToken: '',
 	tokenPostfix: '.scss',
 
@@ -65,32 +59,32 @@ export var language = <IMonarchLanguage>{
 			{ include: '@import' },
 			{ include: '@variabledeclaration' },
 			{ include: '@warndebug' }, // sass: log statements
-			['[@](include)', { token: TOKEN_AT_KEYWORD, next: '@includedeclaration' }], // sass: include statement
-			['[@](keyframes|-webkit-keyframes|-moz-keyframes|-o-keyframes)', { token: TOKEN_AT_KEYWORD, next: '@keyframedeclaration' }],
-			['[@](page|content|font-face|-moz-document)', { token: TOKEN_AT_KEYWORD }], // sass: placeholder for includes
-			['[@](charset|namespace)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }],
-			['[@](function)', { token: TOKEN_AT_KEYWORD, next: '@functiondeclaration' }],
-			['[@](mixin)', { token: TOKEN_AT_KEYWORD, next: '@mixindeclaration' }],
+			['[@](include)', { token: 'keyword', next: '@includedeclaration' }], // sass: include statement
+			['[@](keyframes|-webkit-keyframes|-moz-keyframes|-o-keyframes)', { token: 'keyword', next: '@keyframedeclaration' }],
+			['[@](page|content|font-face|-moz-document)', { token: 'keyword' }], // sass: placeholder for includes
+			['[@](charset|namespace)', { token: 'keyword', next: '@declarationbody' }],
+			['[@](function)', { token: 'keyword', next: '@functiondeclaration' }],
+			['[@](mixin)', { token: 'keyword', next: '@mixindeclaration' }],
 			['url(\\-prefix)?\\(', { token: 'meta', next: '@urldeclaration' }],
 			{ include: '@controlstatement' }, // sass control statements
 			{ include: '@selectorname' },
-			['[&\\*]', TOKEN_SELECTOR_TAG], // selector symbols
+			['[&\\*]', 'tag'], // selector symbols
 			['[>\\+,]', 'delimiter'], // selector operators
 			['\\[', { token: 'delimiter.bracket', next: '@selectorattribute' }],
 			['{', { token: 'delimiter.curly', next: '@selectorbody' }],
 		],
 
 		selectorbody: [
-			['[*_]?@identifier@ws:(?=(\\s|\\d|[^{;}]*[;}]))', TOKEN_PROPERTY, '@rulevalue'], // rule definition: to distinguish from a nested selector check for whitespace, number or a semicolon
+			['[*_]?@identifier@ws:(?=(\\s|\\d|[^{;}]*[;}]))', 'attribute.name', '@rulevalue'], // rule definition: to distinguish from a nested selector check for whitespace, number or a semicolon
 			{ include: '@selector' }, // sass: nested selectors
-			['[@](extend)', { token: TOKEN_AT_KEYWORD, next: '@extendbody' }], // sass: extend other selectors
-			['[@](return)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }],
+			['[@](extend)', { token: 'keyword', next: '@extendbody' }], // sass: extend other selectors
+			['[@](return)', { token: 'keyword', next: '@declarationbody' }],
 			['}', { token: 'delimiter.curly', next: '@pop' }],
 		],
 
 		selectorname: [
 			['#{', { token: 'meta', next: '@variableinterpolation' }], // sass: interpolation
-			['(\\.|#(?=[^{])|%|(@identifier)|:)+', TOKEN_SELECTOR], // selector (.foo, div, ...)
+			['(\\.|#(?=[^{])|%|(@identifier)|:)+', 'tag'], // selector (.foo, div, ...)
 		],
 
 		selectorattribute: [
@@ -122,17 +116,17 @@ export var language = <IMonarchLanguage>{
 		],
 
 		nestedproperty: [
-			['[*_]?@identifier@ws:', TOKEN_PROPERTY, '@rulevalue'],
+			['[*_]?@identifier@ws:', 'attribute.name', '@rulevalue'],
 			{ include: '@comments' },
 			['}', { token: 'delimiter.curly', next: '@pop' }],
 		],
 
 		warndebug: [
-			['[@](warn|debug)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }],
+			['[@](warn|debug)', { token: 'keyword', next: '@declarationbody' }],
 		],
 
 		import: [
-			['[@](import)', { token: TOKEN_AT_KEYWORD, next: '@declarationbody' }],
+			['[@](import)', { token: 'keyword', next: '@declarationbody' }],
 		],
 
 		variabledeclaration: [ // sass variables
@@ -185,7 +179,7 @@ export var language = <IMonarchLanguage>{
 		],
 
 		name: [
-			['@identifier', TOKEN_VALUE],
+			['@identifier', 'attribute.value'],
 		],
 
 		numbers: [
@@ -248,7 +242,7 @@ export var language = <IMonarchLanguage>{
 		],
 
 		functionbody: [
-			['[@](return)', { token: TOKEN_AT_KEYWORD }],
+			['[@](return)', { token: 'keyword' }],
 			{ include: '@variabledeclaration' },
 			{ include: '@term' },
 			{ include: '@controlstatement' },
@@ -261,7 +255,7 @@ export var language = <IMonarchLanguage>{
 		],
 
 		functionarguments: [
-			['\\$@identifier@ws:', TOKEN_PROPERTY],
+			['\\$@identifier@ws:', 'attribute.name'],
 			['[,]', 'delimiter'],
 			{ include: '@term' },
 			['\\)', { token: 'meta', next: '@pop' }],

src/solidity.ts

@@ -3,7 +3,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['/*', '*/'],
@@ -17,7 +17,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.sol',
 

src/sql.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '--',
 		blockComment: ['/*', '*/'],
@@ -34,7 +34,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.sql',
 	ignoreCase: true,

src/swift.ts

@@ -7,7 +7,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '//',
 		blockComment: ['/*', '*/'],
@@ -35,7 +35,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.swift',
 

src/vb.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		lineComment: '\'',
 		blockComment: ['/*', '*/'],
@@ -50,7 +50,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.vb',
 	ignoreCase: true,

src/xml.ts

@@ -8,7 +8,7 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-export var conf: IRichLanguageConfiguration = {
+export const conf: IRichLanguageConfiguration = {
 	comments: {
 		blockComment: ['<!--', '-->'],
 	},
@@ -27,7 +27,7 @@ export var conf: IRichLanguageConfiguration = {
 	]
 };
 
-export var language = <ILanguage>{
+export const language = <ILanguage>{
 	defaultToken: '',
 	tokenPostfix: '.xml',
 

test/handlebars.test.ts

@@ -6,23 +6,6 @@
 'use strict';
 
 import { testTokenization } from './testRunner';
-import { htmlTokenTypes } from '../src/handlebars';
-
-const HTML_DELIM_START = htmlTokenTypes.DELIM_START;
-const HTML_DELIM_END = htmlTokenTypes.DELIM_END;
-const DELIM_ASSIGN = 'delimiter';
-const HTML_ATTRIB_NAME = 'attribute.name';
-const HTML_ATTRIB_VALUE = 'attribute.value';
-function getTag(name: string) {
-	return htmlTokenTypes.getTag(name);
-}
-
-const handlebarsTokenTypes = {
-	EMBED: 'delimiter.handlebars',
-	EMBED_UNESCAPED: 'delimiter.handlebars',
-	KEYWORD: 'keyword.helper.handlebars',
-	VARIABLE: 'variable.parameter.handlebars',
-}
 
 testTokenization(['handlebars', 'css'], [
 
@@ -30,13 +13,13 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<h1>handlebars!</h1>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('h1') },
-			{ startIndex: 3, type: HTML_DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 3, type: 'delimiter.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 15, type: HTML_DELIM_START },
-			{ startIndex: 17, type: getTag('h1') },
-			{ startIndex: 19, type: HTML_DELIM_END }
+			{ startIndex: 15, type: 'delimiter.html' },
+			{ startIndex: 17, type: 'tag.html' },
+			{ startIndex: 19, type: 'delimiter.html' }
 		]
 	}],
 
@@ -44,17 +27,17 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<h1>{{ title }}</h1>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('h1') },
-			{ startIndex: 3, type: HTML_DELIM_END },
-			{ startIndex: 4, type: handlebarsTokenTypes.EMBED },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 3, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'delimiter.handlebars' },
 			{ startIndex: 6, type: '' },
-			{ startIndex: 7, type: handlebarsTokenTypes.VARIABLE },
+			{ startIndex: 7, type: 'variable.parameter.handlebars' },
 			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 15, type: HTML_DELIM_START },
-			{ startIndex: 17, type: getTag('h1') },
-			{ startIndex: 19, type: HTML_DELIM_END }
+			{ startIndex: 13, type: 'delimiter.handlebars' },
+			{ startIndex: 15, type: 'delimiter.html' },
+			{ startIndex: 17, type: 'tag.html' },
+			{ startIndex: 19, type: 'delimiter.html' }
 		]
 	}],
 
@@ -62,15 +45,15 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<h1>{{title}}</h1>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('h1') },
-			{ startIndex: 3, type: HTML_DELIM_END },
-			{ startIndex: 4, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 6, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 11, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 13, type: HTML_DELIM_START },
-			{ startIndex: 15, type: getTag('h1') },
-			{ startIndex: 17, type: HTML_DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 3, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'delimiter.handlebars' },
+			{ startIndex: 6, type: 'variable.parameter.handlebars' },
+			{ startIndex: 11, type: 'delimiter.handlebars' },
+			{ startIndex: 13, type: 'delimiter.html' },
+			{ startIndex: 15, type: 'tag.html' },
+			{ startIndex: 17, type: 'delimiter.html' }
 		]
 	}],
 
@@ -78,17 +61,17 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<h1>{{{ title }}}</h1>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('h1') },
-			{ startIndex: 3, type: HTML_DELIM_END },
-			{ startIndex: 4, type: handlebarsTokenTypes.EMBED_UNESCAPED },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 3, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'delimiter.handlebars' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: handlebarsTokenTypes.VARIABLE },
+			{ startIndex: 8, type: 'variable.parameter.handlebars' },
 			{ startIndex: 13, type: '' },
-			{ startIndex: 14, type: handlebarsTokenTypes.EMBED_UNESCAPED },
-			{ startIndex: 17, type: HTML_DELIM_START },
-			{ startIndex: 19, type: getTag('h1') },
-			{ startIndex: 21, type: HTML_DELIM_END }
+			{ startIndex: 14, type: 'delimiter.handlebars' },
+			{ startIndex: 17, type: 'delimiter.html' },
+			{ startIndex: 19, type: 'tag.html' },
+			{ startIndex: 21, type: 'delimiter.html' }
 		]
 	}],
 
@@ -96,29 +79,29 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<ul>{{#each items}}<li>{{item}}</li>{{/each}}</ul>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('ul') },
-			{ startIndex: 3, type: HTML_DELIM_END },
-			{ startIndex: 4, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 6, type: handlebarsTokenTypes.KEYWORD },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 3, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'delimiter.handlebars' },
+			{ startIndex: 6, type: 'keyword.helper.handlebars' },
 			{ startIndex: 11, type: '' },
-			{ startIndex: 12, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 17, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 19, type: HTML_DELIM_START },
-			{ startIndex: 20, type: getTag('li') },
-			{ startIndex: 22, type: HTML_DELIM_END },
-			{ startIndex: 23, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 25, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 29, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 31, type: HTML_DELIM_START },
-			{ startIndex: 33, type: getTag('li') },
-			{ startIndex: 35, type: HTML_DELIM_END },
-			{ startIndex: 36, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 38, type: handlebarsTokenTypes.KEYWORD },
-			{ startIndex: 43, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 45, type: HTML_DELIM_START },
-			{ startIndex: 47, type: getTag('ul') },
-			{ startIndex: 49, type: HTML_DELIM_END }
+			{ startIndex: 12, type: 'variable.parameter.handlebars' },
+			{ startIndex: 17, type: 'delimiter.handlebars' },
+			{ startIndex: 19, type: 'delimiter.html' },
+			{ startIndex: 20, type: 'tag.html' },
+			{ startIndex: 22, type: 'delimiter.html' },
+			{ startIndex: 23, type: 'delimiter.handlebars' },
+			{ startIndex: 25, type: 'variable.parameter.handlebars' },
+			{ startIndex: 29, type: 'delimiter.handlebars' },
+			{ startIndex: 31, type: 'delimiter.html' },
+			{ startIndex: 33, type: 'tag.html' },
+			{ startIndex: 35, type: 'delimiter.html' },
+			{ startIndex: 36, type: 'delimiter.handlebars' },
+			{ startIndex: 38, type: 'keyword.helper.handlebars' },
+			{ startIndex: 43, type: 'delimiter.handlebars' },
+			{ startIndex: 45, type: 'delimiter.html' },
+			{ startIndex: 47, type: 'tag.html' },
+			{ startIndex: 49, type: 'delimiter.html' }
 		]
 	}],
 
@@ -126,38 +109,38 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<div>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('div') },
-			{ startIndex: 4, type: HTML_DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 4, type: 'delimiter.html' }
 		]
 	}, {
 		line: '{{#if foo}}',
 		tokens: [
-			{ startIndex: 0, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 2, type: handlebarsTokenTypes.KEYWORD },
+			{ startIndex: 0, type: 'delimiter.handlebars' },
+			{ startIndex: 2, type: 'keyword.helper.handlebars' },
 			{ startIndex: 5, type: '' },
-			{ startIndex: 6, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 9, type: handlebarsTokenTypes.EMBED }
+			{ startIndex: 6, type: 'variable.parameter.handlebars' },
+			{ startIndex: 9, type: 'delimiter.handlebars' }
 		]
 	}, {
 		line: '<span>{{bar}}</span>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('span') },
-			{ startIndex: 5, type: HTML_DELIM_END },
-			{ startIndex: 6, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 8, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 11, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 13, type: HTML_DELIM_START },
-			{ startIndex: 15, type: getTag('span') },
-			{ startIndex: 19, type: HTML_DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 5, type: 'delimiter.html' },
+			{ startIndex: 6, type: 'delimiter.handlebars' },
+			{ startIndex: 8, type: 'variable.parameter.handlebars' },
+			{ startIndex: 11, type: 'delimiter.handlebars' },
+			{ startIndex: 13, type: 'delimiter.html' },
+			{ startIndex: 15, type: 'tag.html' },
+			{ startIndex: 19, type: 'delimiter.html' }
 		]
 	}, {
 		line: '{{/if}}',
 		tokens: [
-			{ startIndex: 0, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 2, type: handlebarsTokenTypes.KEYWORD },
-			{ startIndex: 5, type: handlebarsTokenTypes.EMBED }
+			{ startIndex: 0, type: 'delimiter.handlebars' },
+			{ startIndex: 2, type: 'keyword.helper.handlebars' },
+			{ startIndex: 5, type: 'delimiter.handlebars' }
 		]
 	}],
 
@@ -165,9 +148,9 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '</div>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 2, type: getTag('div') },
-			{ startIndex: 5, type: HTML_DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -175,27 +158,27 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<script type="text/x-handlebars-template"><h1>{{ title }}</h1></script>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: HTML_ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: HTML_ATTRIB_VALUE },
-			{ startIndex: 41, type: HTML_DELIM_END },
-			{ startIndex: 42, type: HTML_DELIM_START },
-			{ startIndex: 43, type: getTag('h1') },
-			{ startIndex: 45, type: HTML_DELIM_END },
-			{ startIndex: 46, type: handlebarsTokenTypes.EMBED },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 41, type: 'delimiter.html' },
+			{ startIndex: 42, type: 'delimiter.html' },
+			{ startIndex: 43, type: 'tag.html' },
+			{ startIndex: 45, type: 'delimiter.html' },
+			{ startIndex: 46, type: 'delimiter.handlebars' },
 			{ startIndex: 48, type: '' },
-			{ startIndex: 49, type: handlebarsTokenTypes.VARIABLE },
+			{ startIndex: 49, type: 'variable.parameter.handlebars' },
 			{ startIndex: 54, type: '' },
-			{ startIndex: 55, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 57, type: HTML_DELIM_START },
-			{ startIndex: 59, type: getTag('h1') },
-			{ startIndex: 61, type: HTML_DELIM_END },
-			{ startIndex: 62, type: HTML_DELIM_START },
-			{ startIndex: 64, type: getTag('script') },
-			{ startIndex: 70, type: HTML_DELIM_END }
+			{ startIndex: 55, type: 'delimiter.handlebars' },
+			{ startIndex: 57, type: 'delimiter.html' },
+			{ startIndex: 59, type: 'tag.html' },
+			{ startIndex: 61, type: 'delimiter.html' },
+			{ startIndex: 62, type: 'delimiter.html' },
+			{ startIndex: 64, type: 'tag.html' },
+			{ startIndex: 70, type: 'delimiter.html' }
 		]
 	}],
 
@@ -203,35 +186,35 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<script type="text/x-handlebars-template">',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: HTML_ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: HTML_ATTRIB_VALUE },
-			{ startIndex: 41, type: HTML_DELIM_END }
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 41, type: 'delimiter.html' }
 		]
 	}, {
 		line: '<h1>{{ title }}</h1>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('h1') },
-			{ startIndex: 3, type: HTML_DELIM_END },
-			{ startIndex: 4, type: handlebarsTokenTypes.EMBED },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 3, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'delimiter.handlebars' },
 			{ startIndex: 6, type: '' },
-			{ startIndex: 7, type: handlebarsTokenTypes.VARIABLE },
+			{ startIndex: 7, type: 'variable.parameter.handlebars' },
 			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 15, type: HTML_DELIM_START },
-			{ startIndex: 17, type: getTag('h1') },
-			{ startIndex: 19, type: HTML_DELIM_END }
+			{ startIndex: 13, type: 'delimiter.handlebars' },
+			{ startIndex: 15, type: 'delimiter.html' },
+			{ startIndex: 17, type: 'tag.html' },
+			{ startIndex: 19, type: 'delimiter.html' }
 		]
 	}, {
 		line: '</script>',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 2, type: getTag('script') },
-			{ startIndex: 8, type: HTML_DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -239,18 +222,18 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '{{foo}}<script></script>{{bar}}',
 		tokens: [
-			{ startIndex: 0, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 2, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 5, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 7, type: HTML_DELIM_START },
-			{ startIndex: 8, type: getTag('script') },
-			{ startIndex: 14, type: HTML_DELIM_END },
-			// { startIndex:15, type: HTML_DELIM_START },
-			{ startIndex: 17, type: getTag('script') },
-			{ startIndex: 23, type: HTML_DELIM_END },
-			{ startIndex: 24, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 26, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 29, type: handlebarsTokenTypes.EMBED }
+			{ startIndex: 0, type: 'delimiter.handlebars' },
+			{ startIndex: 2, type: 'variable.parameter.handlebars' },
+			{ startIndex: 5, type: 'delimiter.handlebars' },
+			{ startIndex: 7, type: 'delimiter.html' },
+			{ startIndex: 8, type: 'tag.html' },
+			{ startIndex: 14, type: 'delimiter.html' },
+			// { startIndex:15, type: 'delimiter.html' },
+			{ startIndex: 17, type: 'tag.html' },
+			{ startIndex: 23, type: 'delimiter.html' },
+			{ startIndex: 24, type: 'delimiter.handlebars' },
+			{ startIndex: 26, type: 'variable.parameter.handlebars' },
+			{ startIndex: 29, type: 'delimiter.handlebars' }
 		]
 	}],
 
@@ -258,9 +241,9 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '{{else}}',
 		tokens: [
-			{ startIndex: 0, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 2, type: handlebarsTokenTypes.KEYWORD },
-			{ startIndex: 6, type: handlebarsTokenTypes.EMBED }
+			{ startIndex: 0, type: 'delimiter.handlebars' },
+			{ startIndex: 2, type: 'keyword.helper.handlebars' },
+			{ startIndex: 6, type: 'delimiter.handlebars' }
 		]
 	}],
 
@@ -268,9 +251,9 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '{{elseFoo}}',
 		tokens: [
-			{ startIndex: 0, type: handlebarsTokenTypes.EMBED },
-			{ startIndex: 2, type: handlebarsTokenTypes.VARIABLE },
-			{ startIndex: 9, type: handlebarsTokenTypes.EMBED }
+			{ startIndex: 0, type: 'delimiter.handlebars' },
+			{ startIndex: 2, type: 'variable.parameter.handlebars' },
+			{ startIndex: 9, type: 'delimiter.handlebars' }
 		]
 	}],
 
@@ -278,13 +261,13 @@ testTokenization(['handlebars', 'css'], [
 	[{
 		line: '<a href="/posts/{{permalink}}">',
 		tokens: [
-			{ startIndex: 0, type: HTML_DELIM_START },
-			{ startIndex: 1, type: getTag('a') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 2, type: '' },
-			{ startIndex: 3, type: HTML_ATTRIB_NAME },
-			{ startIndex: 7, type: DELIM_ASSIGN },
-			{ startIndex: 8, type: HTML_ATTRIB_VALUE },
-			{ startIndex: 30, type: HTML_DELIM_END }
+			{ startIndex: 3, type: 'attribute.name' },
+			{ startIndex: 7, type: 'delimiter' },
+			{ startIndex: 8, type: 'attribute.value' },
+			{ startIndex: 30, type: 'delimiter.html' }
 		]
 	}]
 ]);

test/html.test.ts

@@ -6,22 +6,6 @@
 'use strict';
 
 import { testTokenization } from './testRunner';
-import { htmlTokenTypes } from '../src/html';
-
-const DELIM_START = htmlTokenTypes.DELIM_START + '.html';
-const DELIM_END = htmlTokenTypes.DELIM_END + '.html';
-const ATTRIB_NAME = 'attribute.name.html';
-const DELIM_ASSIGN = 'delimiter.html';
-const ATTRIB_VALUE = 'attribute.value.html';
-
-function getTag(name: string): string {
-	return htmlTokenTypes.getTag(name) + '.html';
-}
-
-const DELIM_COMMENT = 'comment.html';
-const COMMENT = 'comment.content.html';
-const DELIM_DOCTYPE = 'metatag.html';
-const DOCTYPE = 'metatag.content.html';
 
 testTokenization(['html', 'css'], [
 
@@ -29,8 +13,8 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' }
 		]
 	}],
 
@@ -38,8 +22,8 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<input',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('input') }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' }
 		]
 	}],
 
@@ -47,7 +31,7 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '< abc',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
+			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: '' }
 		]
 	}],
@@ -56,7 +40,7 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '< abc>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
+			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: '' }
 		]
 	}],
@@ -66,8 +50,8 @@ testTokenization(['html', 'css'], [
 		line: 'i <len;',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 2, type: DELIM_START },
-			{ startIndex: 3, type: getTag('len') },
+			{ startIndex: 2, type: 'delimiter.html' },
+			{ startIndex: 3, type: 'tag.html' },
 			{ startIndex: 6, type: '' }
 		]
 	}],
@@ -76,7 +60,7 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START }
+			{ startIndex: 0, type: 'delimiter.html' }
 		]
 	}],
 
@@ -84,8 +68,8 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '</a',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 2, type: getTag('a') }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' }
 		]
 	}],
 
@@ -93,9 +77,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
-			{ startIndex: 4, type: DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 4, type: 'delimiter.html' }
 		]
 	}],
 
@@ -103,10 +87,10 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc >',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: DELIM_END }
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -114,9 +98,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<foo:bar>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('foo-bar') },
-			{ startIndex: 8, type: DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -124,9 +108,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '</abc>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 2, type: getTag('abc') },
-			{ startIndex: 5, type: DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -134,10 +118,10 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '</abc  >',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 2, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
 			{ startIndex: 5, type: '' },
-			{ startIndex: 7, type: DELIM_END }
+			{ startIndex: 7, type: 'delimiter.html' }
 		]
 	}],
 
@@ -145,10 +129,10 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc />',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: DELIM_END }
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -156,17 +140,17 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/javascript">var i= 10;</script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
-			{ startIndex: 30, type: DELIM_END },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' },
 			{ startIndex: 31, type: '' },
-			{ startIndex: 41, type: DELIM_START },
-			{ startIndex: 43, type: getTag('script') },
-			{ startIndex: 49, type: DELIM_END }
+			{ startIndex: 41, type: 'delimiter.html' },
+			{ startIndex: 43, type: 'tag.html' },
+			{ startIndex: 49, type: 'delimiter.html' }
 		]
 	}],
 
@@ -174,13 +158,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/javascript">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
-			{ startIndex: 30, type: DELIM_END }
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' }
 		]
 	}, {
 		line: 'var i= 10;',
@@ -190,9 +174,9 @@ testTokenization(['html', 'css'], [
 	}, {
 		line: '</script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 2, type: getTag('script') },
-			{ startIndex: 8, type: DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -200,21 +184,21 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/javascript">var i= 10;',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
-			{ startIndex: 30, type: DELIM_END },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' },
 			{ startIndex: 31, type: '' },
 		]
 	}, {
 		line: '</script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 2, type: getTag('script') },
-			{ startIndex: 8, type: DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -222,21 +206,21 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/javascript">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
-			{ startIndex: 30, type: DELIM_END }
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' }
 		]
 	}, {
 		line: 'var i= 10;</script>',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 10, type: DELIM_START },
-			{ startIndex: 12, type: getTag('script') },
-			{ startIndex: 18, type: DELIM_END }
+			{ startIndex: 10, type: 'delimiter.html' },
+			{ startIndex: 12, type: 'tag.html' },
+			{ startIndex: 18, type: 'delimiter.html' }
 		]
 	}],
 
@@ -244,22 +228,22 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/plain">a',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
-			{ startIndex: 25, type: DELIM_END },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 25, type: 'delimiter.html' },
 			{ startIndex: 26, type: '' },
 		]
 	}, {
 		line: '<a</script>',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 2, type: DELIM_START },
-			{ startIndex: 4, type: getTag('script') },
-			{ startIndex: 10, type: DELIM_END }
+			{ startIndex: 2, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'tag.html' },
+			{ startIndex: 10, type: 'delimiter.html' }
 		]
 	}],
 
@@ -267,20 +251,20 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script>a</script><script>b</script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
-			{ startIndex: 7, type: DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 7, type: 'delimiter.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 9, type: DELIM_START },
-			{ startIndex: 11, type: getTag('script') },
-			{ startIndex: 17, type: DELIM_END },
-			// { startIndex:18, type: DELIM_START },
-			{ startIndex: 19, type: getTag('script') },
-			{ startIndex: 25, type: DELIM_END },
+			{ startIndex: 9, type: 'delimiter.html' },
+			{ startIndex: 11, type: 'tag.html' },
+			{ startIndex: 17, type: 'delimiter.html' },
+			// { startIndex:18, type: 'delimiter.html' },
+			{ startIndex: 19, type: 'tag.html' },
+			{ startIndex: 25, type: 'delimiter.html' },
 			{ startIndex: 26, type: '' },
-			{ startIndex: 27, type: DELIM_START },
-			{ startIndex: 29, type: getTag('script') },
-			{ startIndex: 35, type: DELIM_END }
+			{ startIndex: 27, type: 'delimiter.html' },
+			{ startIndex: 29, type: 'tag.html' },
+			{ startIndex: 35, type: 'delimiter.html' }
 		]
 	}],
 
@@ -288,16 +272,16 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/javascript"></script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
-			{ startIndex: 30, type: DELIM_END },
-			// { startIndex:31, type: DELIM_START },
-			{ startIndex: 33, type: getTag('script') },
-			{ startIndex: 39, type: DELIM_END }
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' },
+			// { startIndex:31, type: 'delimiter.html' },
+			{ startIndex: 33, type: 'tag.html' },
+			{ startIndex: 39, type: 'delimiter.html' }
 		]
 	}],
 
@@ -305,13 +289,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script>var i= 10;</script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
-			{ startIndex: 7, type: DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 7, type: 'delimiter.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 18, type: DELIM_START },
-			{ startIndex: 20, type: getTag('script') },
-			{ startIndex: 26, type: DELIM_END }
+			{ startIndex: 18, type: 'delimiter.html' },
+			{ startIndex: 20, type: 'tag.html' },
+			{ startIndex: 26, type: 'delimiter.html' }
 		]
 	}],
 
@@ -319,20 +303,20 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<script type="text/javascript" src="main.js"></script>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('script') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_ASSIGN },
-			{ startIndex: 13, type: ATTRIB_VALUE },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
 			{ startIndex: 30, type: '' },
-			{ startIndex: 31, type: ATTRIB_NAME },
-			{ startIndex: 34, type: DELIM_ASSIGN },
-			{ startIndex: 35, type: ATTRIB_VALUE },
-			{ startIndex: 44, type: DELIM_END },
-			// { startIndex:45, type: DELIM_START },
-			{ startIndex: 47, type: getTag('script') },
-			{ startIndex: 53, type: DELIM_END }
+			{ startIndex: 31, type: 'attribute.name.html' },
+			{ startIndex: 34, type: 'delimiter.html' },
+			{ startIndex: 35, type: 'attribute.value.html' },
+			{ startIndex: 44, type: 'delimiter.html' },
+			// { startIndex:45, type: 'delimiter.html' },
+			{ startIndex: 47, type: 'tag.html' },
+			{ startIndex: 53, type: 'delimiter.html' }
 		]
 	}],
 
@@ -340,13 +324,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo="bar">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
-			{ startIndex: 9, type: ATTRIB_VALUE },
-			{ startIndex: 14, type: DELIM_END }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
+			{ startIndex: 14, type: 'delimiter.html' }
 		]
 	}],
 
@@ -354,13 +338,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo=\'bar\'>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
-			{ startIndex: 9, type: ATTRIB_VALUE },
-			{ startIndex: 14, type: DELIM_END }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
+			{ startIndex: 14, type: 'delimiter.html' }
 		]
 	}],
 
@@ -368,13 +352,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo="">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
-			{ startIndex: 9, type: ATTRIB_VALUE },
-			{ startIndex: 11, type: DELIM_END }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
+			{ startIndex: 11, type: 'delimiter.html' }
 		]
 	}],
 
@@ -382,17 +366,17 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo="bar" bar=\'foo\'>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
-			{ startIndex: 9, type: ATTRIB_VALUE },
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
 			{ startIndex: 14, type: '' },
-			{ startIndex: 15, type: ATTRIB_NAME },
-			{ startIndex: 18, type: DELIM_ASSIGN },
-			{ startIndex: 19, type: ATTRIB_VALUE },
-			{ startIndex: 24, type: DELIM_END }
+			{ startIndex: 15, type: 'attribute.name.html' },
+			{ startIndex: 18, type: 'delimiter.html' },
+			{ startIndex: 19, type: 'attribute.value.html' },
+			{ startIndex: 24, type: 'delimiter.html' }
 		]
 	}],
 
@@ -400,17 +384,17 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo=bar bar=help-me>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
-			{ startIndex: 9, type: ATTRIB_NAME }, // slightly incorrect
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.name.html' }, // slightly incorrect
 			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: ATTRIB_NAME },
-			{ startIndex: 16, type: DELIM_ASSIGN },
-			{ startIndex: 17, type: ATTRIB_NAME }, // slightly incorrect
-			{ startIndex: 24, type: DELIM_END }
+			{ startIndex: 13, type: 'attribute.name.html' },
+			{ startIndex: 16, type: 'delimiter.html' },
+			{ startIndex: 17, type: 'attribute.name.html' }, // slightly incorrect
+			{ startIndex: 24, type: 'delimiter.html' }
 		]
 	}],
 
@@ -418,14 +402,14 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo=  "bar">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
 			{ startIndex: 9, type: '' },
-			{ startIndex: 11, type: ATTRIB_VALUE },
-			{ startIndex: 16, type: DELIM_END }
+			{ startIndex: 11, type: 'attribute.value.html' },
+			{ startIndex: 16, type: 'delimiter.html' }
 		]
 	}],
 
@@ -433,15 +417,15 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo = "bar">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
+			{ startIndex: 5, type: 'attribute.name.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 9, type: DELIM_ASSIGN },
+			{ startIndex: 9, type: 'delimiter.html' },
 			{ startIndex: 10, type: '' },
-			{ startIndex: 11, type: ATTRIB_VALUE },
-			{ startIndex: 16, type: DELIM_END }
+			{ startIndex: 11, type: 'attribute.value.html' },
+			{ startIndex: 16, type: 'delimiter.html' }
 		]
 	}],
 
@@ -449,11 +433,11 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_END }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -461,13 +445,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo bar>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
+			{ startIndex: 5, type: 'attribute.name.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 9, type: ATTRIB_NAME },
-			{ startIndex: 12, type: DELIM_END }
+			{ startIndex: 9, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' }
 		]
 	}],
 
@@ -475,14 +459,14 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo!@#="bar">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
+			{ startIndex: 5, type: 'attribute.name.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 11, type: DELIM_ASSIGN },
-			{ startIndex: 12, type: ATTRIB_VALUE },
-			{ startIndex: 17, type: DELIM_END }
+			{ startIndex: 11, type: 'delimiter.html' },
+			{ startIndex: 12, type: 'attribute.value.html' },
+			{ startIndex: 17, type: 'delimiter.html' }
 		]
 	}],
 
@@ -490,25 +474,25 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc #myinput (click)="bar" [value]="someProperty" *ngIf="someCondition">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 6, type: ATTRIB_NAME },
+			{ startIndex: 6, type: 'attribute.name.html' },
 			{ startIndex: 13, type: '' },
-			{ startIndex: 15, type: ATTRIB_NAME },
+			{ startIndex: 15, type: 'attribute.name.html' },
 			{ startIndex: 20, type: '' },
-			{ startIndex: 21, type: DELIM_ASSIGN },
-			{ startIndex: 22, type: ATTRIB_VALUE },
+			{ startIndex: 21, type: 'delimiter.html' },
+			{ startIndex: 22, type: 'attribute.value.html' },
 			{ startIndex: 27, type: '' },
-			{ startIndex: 29, type: ATTRIB_NAME },
+			{ startIndex: 29, type: 'attribute.name.html' },
 			{ startIndex: 34, type: '' },
-			{ startIndex: 35, type: DELIM_ASSIGN },
-			{ startIndex: 36, type: ATTRIB_VALUE },
+			{ startIndex: 35, type: 'delimiter.html' },
+			{ startIndex: 36, type: 'attribute.value.html' },
 			{ startIndex: 50, type: '' },
-			{ startIndex: 52, type: ATTRIB_NAME },
-			{ startIndex: 56, type: DELIM_ASSIGN },
-			{ startIndex: 57, type: ATTRIB_VALUE },
-			{ startIndex: 72, type: DELIM_END }
+			{ startIndex: 52, type: 'attribute.name.html' },
+			{ startIndex: 56, type: 'delimiter.html' },
+			{ startIndex: 57, type: 'attribute.value.html' },
+			{ startIndex: 72, type: 'delimiter.html' }
 		]
 	}],
 
@@ -516,13 +500,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<abc foo=">',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('abc') },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: ATTRIB_NAME },
-			{ startIndex: 8, type: DELIM_ASSIGN },
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
 			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: DELIM_END }
+			{ startIndex: 10, type: 'delimiter.html' }
 		]
 	}],
 
@@ -530,9 +514,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<!--a-->',
 		tokens: [
-			{ startIndex: 0, type: DELIM_COMMENT },
-			{ startIndex: 4, type: COMMENT },
-			{ startIndex: 5, type: DELIM_COMMENT }
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' },
+			{ startIndex: 5, type: 'comment.html' }
 		]
 	}],
 
@@ -540,9 +524,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<!--a>foo bar</a -->',
 		tokens: [
-			{ startIndex: 0, type: DELIM_COMMENT },
-			{ startIndex: 4, type: COMMENT },
-			{ startIndex: 17, type: DELIM_COMMENT }
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' },
+			{ startIndex: 17, type: 'comment.html' }
 		]
 	}],
 
@@ -550,19 +534,19 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<!--a>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_COMMENT },
-			{ startIndex: 4, type: COMMENT }
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' }
 		]
 	}, {
 		line: 'foo ',
 		tokens: [
-			{ startIndex: 0, type: COMMENT },
+			{ startIndex: 0, type: 'comment.content.html' },
 		]
 	}, {
 		line: 'bar</a -->',
 		tokens: [
-			{ startIndex: 0, type: COMMENT },
-			{ startIndex: 7, type: DELIM_COMMENT }
+			{ startIndex: 0, type: 'comment.content.html' },
+			{ startIndex: 7, type: 'comment.html' }
 		]
 	}],
 
@@ -570,9 +554,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<!DOCTYPE a>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_DOCTYPE },
-			{ startIndex: 9, type: DOCTYPE },
-			{ startIndex: 11, type: DELIM_DOCTYPE }
+			{ startIndex: 0, type: 'metatag.html' },
+			{ startIndex: 9, type: 'metatag.content.html' },
+			{ startIndex: 11, type: 'metatag.html' }
 		]
 	}],
 
@@ -580,9 +564,9 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<!doctype a>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_DOCTYPE },
-			{ startIndex: 9, type: DOCTYPE },
-			{ startIndex: 11, type: DELIM_DOCTYPE }
+			{ startIndex: 0, type: 'metatag.html' },
+			{ startIndex: 9, type: 'metatag.content.html' },
+			{ startIndex: 11, type: 'metatag.html' }
 		]
 	}],
 
@@ -590,14 +574,14 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<!DOCTYPE a',
 		tokens: [
-			{ startIndex: 0, type: DELIM_DOCTYPE },
-			{ startIndex: 9, type: DOCTYPE },
+			{ startIndex: 0, type: 'metatag.html' },
+			{ startIndex: 9, type: 'metatag.content.html' },
 		]
 	}, {
 		line: '"foo" \'bar\'>',
 		tokens: [
-			{ startIndex: 0, type: DOCTYPE },
-			{ startIndex: 11, type: DELIM_DOCTYPE }
+			{ startIndex: 0, type: 'metatag.content.html' },
+			{ startIndex: 11, type: 'metatag.html' }
 		]
 	}],
 
@@ -605,13 +589,13 @@ testTokenization(['html', 'css'], [
 	[{
 		line: '<asdf:bar>asd</asdf:bar>',
 		tokens: [
-			{ startIndex: 0, type: DELIM_START },
-			{ startIndex: 1, type: getTag('asdf:bar') },
-			{ startIndex: 9, type: DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 9, type: 'delimiter.html' },
 			{ startIndex: 10, type: '' },
-			{ startIndex: 13, type: DELIM_START },
-			{ startIndex: 15, type: getTag('asdf:bar') },
-			{ startIndex: 23, type: DELIM_END }
+			{ startIndex: 13, type: 'delimiter.html' },
+			{ startIndex: 15, type: 'tag.html' },
+			{ startIndex: 23, type: 'delimiter.html' }
 		]
 	}]
 ]);

test/php.test.ts

@@ -6,7 +6,6 @@
 'use strict';
 
 import { testTokenization } from './testRunner';
-import { htmlTokenTypes } from '../src/php';
 
 testTokenization(['php', 'css'], [
 	// Bug 13596:[ErrorTelemetry] Stream did not advance while tokenizing. Mode id is php (stuck)
@@ -1809,9 +1808,9 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 6, type: 'delimiter.php' },
 			{ startIndex: 7, type: 'number.php' },
 			{ startIndex: 8, type: 'metatag.php' },
-			{ startIndex: 10, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 11, type: htmlTokenTypes.getTag('br') },
-			{ startIndex: 13, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 10, type: 'delimiter.html' },
+			{ startIndex: 11, type: 'tag.html' },
+			{ startIndex: 13, type: 'delimiter.html' },
 			{ startIndex: 15, type: 'metatag.php' },
 			{ startIndex: 18, type: 'number.php' },
 			{ startIndex: 19, type: 'metatag.php' }
@@ -1827,9 +1826,9 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 6, type: 'delimiter.php' },
 			{ startIndex: 7, type: 'number.php' },
 			{ startIndex: 8, type: 'metatag.php' },
-			{ startIndex: 10, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 11, type: htmlTokenTypes.getTag('abc') },
-			{ startIndex: 14, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 10, type: 'delimiter.html' },
+			{ startIndex: 11, type: 'tag.html' },
+			{ startIndex: 14, type: 'delimiter.html' },
 			{ startIndex: 15, type: 'metatag.php' },
 			{ startIndex: 18, type: 'number.php' },
 			{ startIndex: 19, type: 'metatag.php' }
@@ -1840,17 +1839,17 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<abc><?php5+3?><abc>',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('abc') },
-			{ startIndex: 4, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 4, type: 'delimiter.html' },
 			{ startIndex: 5, type: 'metatag.php' },
 			{ startIndex: 10, type: 'number.php' },
 			{ startIndex: 11, type: 'delimiter.php' },
 			{ startIndex: 12, type: 'number.php' },
 			{ startIndex: 13, type: 'metatag.php' },
-			{ startIndex: 15, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 16, type: htmlTokenTypes.getTag('abc') },
-			{ startIndex: 19, type: htmlTokenTypes.DELIM_END }
+			{ startIndex: 15, type: 'delimiter.html' },
+			{ startIndex: 16, type: 'tag.html' },
+			{ startIndex: 19, type: 'delimiter.html' }
 		]
 	}],
 
@@ -1858,24 +1857,24 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<abc><script>var i= 10;</script><?php5+3?><abc>',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('abc') },
-			{ startIndex: 4, type: htmlTokenTypes.DELIM_END },
-			// { startIndex:5, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 6, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 12, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 4, type: 'delimiter.html' },
+			// { startIndex:5, type: 'delimiter.html' },
+			{ startIndex: 6, type: 'tag.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: '' },
-			{ startIndex: 23, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 25, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 31, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 23, type: 'delimiter.html' },
+			{ startIndex: 25, type: 'tag.html' },
+			{ startIndex: 31, type: 'delimiter.html' },
 			{ startIndex: 32, type: 'metatag.php' },
 			{ startIndex: 37, type: 'number.php' },
 			{ startIndex: 38, type: 'delimiter.php' },
 			{ startIndex: 39, type: 'number.php' },
 			{ startIndex: 40, type: 'metatag.php' },
-			{ startIndex: 42, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 43, type: htmlTokenTypes.getTag('abc') },
-			{ startIndex: 46, type: htmlTokenTypes.DELIM_END }
+			{ startIndex: 42, type: 'delimiter.html' },
+			{ startIndex: 43, type: 'tag.html' },
+			{ startIndex: 46, type: 'delimiter.html' }
 		]
 	}],
 
@@ -1883,28 +1882,28 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<abc><script>var i= 10;</script><?php5+3?><script>var x= 15;</script>',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('abc') },
-			{ startIndex: 4, type: htmlTokenTypes.DELIM_END },
-			// { startIndex:5, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 6, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 12, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 4, type: 'delimiter.html' },
+			// { startIndex:5, type: 'delimiter.html' },
+			{ startIndex: 6, type: 'tag.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: '' },
-			{ startIndex: 23, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 25, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 31, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 23, type: 'delimiter.html' },
+			{ startIndex: 25, type: 'tag.html' },
+			{ startIndex: 31, type: 'delimiter.html' },
 			{ startIndex: 32, type: 'metatag.php' },
 			{ startIndex: 37, type: 'number.php' },
 			{ startIndex: 38, type: 'delimiter.php' },
 			{ startIndex: 39, type: 'number.php' },
 			{ startIndex: 40, type: 'metatag.php' },
-			{ startIndex: 42, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 43, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 49, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 42, type: 'delimiter.html' },
+			{ startIndex: 43, type: 'tag.html' },
+			{ startIndex: 49, type: 'delimiter.html' },
 			{ startIndex: 50, type: '' },
-			{ startIndex: 60, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 62, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 68, type: htmlTokenTypes.DELIM_END }
+			{ startIndex: 60, type: 'delimiter.html' },
+			{ startIndex: 62, type: 'tag.html' },
+			{ startIndex: 68, type: 'delimiter.html' }
 		]
 	}],
 
@@ -1912,16 +1911,16 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<html>',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('html') },
-			{ startIndex: 5, type: htmlTokenTypes.DELIM_END }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}, {
 		line: '<style><?="div"?>{ color:blue; }</style>',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('style') },
-			{ startIndex: 6, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 6, type: 'delimiter.html' },
 			{ startIndex: 7, type: 'metatag.php' },
 			{ startIndex: 10, type: 'string.php' },
 			{ startIndex: 15, type: 'metatag.php' },
@@ -1932,9 +1931,9 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 29, type: 'delimiter.css' },
 			{ startIndex: 30, type: '' },
 			{ startIndex: 31, type: 'delimiter.bracket.css' },
-			{ startIndex: 32, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 34, type: htmlTokenTypes.getTag('style') },
-			{ startIndex: 39, type: htmlTokenTypes.DELIM_END }
+			{ startIndex: 32, type: 'delimiter.html' },
+			{ startIndex: 34, type: 'tag.html' },
+			{ startIndex: 39, type: 'delimiter.html' }
 		]
 	}],
 
@@ -1942,12 +1941,12 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<html><style><?="div"?> { color:blue; }</style><!--<?="HTML Comment"?>--><script>var x = 3;/* <?="JS Comment"/*</script>*/?> */var y = 4;</script></html><? $x = 3;?>',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('html') },
-			{ startIndex: 5, type: htmlTokenTypes.DELIM_END },
-			// { startIndex:6, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 7, type: htmlTokenTypes.getTag('style') },
-			{ startIndex: 12, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 5, type: 'delimiter.html' },
+			// { startIndex:6, type: 'delimiter.html' },
+			{ startIndex: 7, type: 'tag.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: 'metatag.php' },
 			{ startIndex: 16, type: 'string.php' },
 			{ startIndex: 21, type: 'metatag.php' },
@@ -1959,29 +1958,29 @@ testTokenization(['php', 'css'], [
 			{ startIndex: 36, type: 'delimiter.css' },
 			{ startIndex: 37, type: '' },
 			{ startIndex: 38, type: 'delimiter.bracket.css' },
-			{ startIndex: 39, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 41, type: htmlTokenTypes.getTag('style') },
-			{ startIndex: 46, type: htmlTokenTypes.DELIM_END },
-			{ startIndex: 47, type: htmlTokenTypes.DELIM_COMMENT },
+			{ startIndex: 39, type: 'delimiter.html' },
+			{ startIndex: 41, type: 'tag.html' },
+			{ startIndex: 46, type: 'delimiter.html' },
+			{ startIndex: 47, type: 'comment.html' },
 			{ startIndex: 51, type: 'metatag.php' },
 			{ startIndex: 54, type: 'string.php' },
 			{ startIndex: 68, type: 'metatag.php' },
-			{ startIndex: 70, type: htmlTokenTypes.DELIM_COMMENT },
-			{ startIndex: 73, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 74, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 80, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 70, type: 'comment.html' },
+			{ startIndex: 73, type: 'delimiter.html' },
+			{ startIndex: 74, type: 'tag.html' },
+			{ startIndex: 80, type: 'delimiter.html' },
 			{ startIndex: 81, type: '' },
 			{ startIndex: 94, type: 'metatag.php' },
 			{ startIndex: 97, type: 'string.php' },
 			{ startIndex: 109, type: 'comment.php' },
 			{ startIndex: 122, type: 'metatag.php' },
 			{ startIndex: 124, type: '' },
-			{ startIndex: 137, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 139, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 145, type: htmlTokenTypes.DELIM_END },
-			// { startIndex:146, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 148, type: htmlTokenTypes.getTag('html') },
-			{ startIndex: 152, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 137, type: 'delimiter.html' },
+			{ startIndex: 139, type: 'tag.html' },
+			{ startIndex: 145, type: 'delimiter.html' },
+			// { startIndex:146, type: 'delimiter.html' },
+			{ startIndex: 148, type: 'tag.html' },
+			{ startIndex: 152, type: 'delimiter.html' },
 			{ startIndex: 153, type: 'metatag.php' },
 			{ startIndex: 155, type: '' },
 			{ startIndex: 156, type: 'variable.php' },
@@ -1998,9 +1997,9 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<!--c--><?',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_COMMENT },
-			{ startIndex: 4, type: htmlTokenTypes.COMMENT },
-			{ startIndex: 5, type: htmlTokenTypes.DELIM_COMMENT },
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' },
+			{ startIndex: 5, type: 'comment.html' },
 			{ startIndex: 8, type: 'metatag.php' }
 		]
 	}],
@@ -2008,9 +2007,9 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<script>//<?',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 7, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 7, type: 'delimiter.html' },
 			{ startIndex: 8, type: '' },
 			{ startIndex: 10, type: 'metatag.php' }
 		]
@@ -2019,9 +2018,9 @@ testTokenization(['php', 'css'], [
 	[{
 		line: '<script>"<?php5+3?>"',
 		tokens: [
-			{ startIndex: 0, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 1, type: htmlTokenTypes.getTag('script') },
-			{ startIndex: 7, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 7, type: 'delimiter.html' },
 			{ startIndex: 8, type: '' },
 			{ startIndex: 9, type: 'metatag.php' },
 			{ startIndex: 14, type: 'number.php' },

test/razor.test.ts

@@ -6,9 +6,6 @@
 'use strict';
 
 import { testTokenization } from './testRunner';
-import { htmlTokenTypes } from '../src/php';
-
-const EMBED_CS = 'metatag.cs';
 
 testTokenization('razor', [
 
@@ -16,22 +13,22 @@ testTokenization('razor', [
 	[{
 		line: '@{ var x; <b>x</b> }',
 		tokens: [
-			{ startIndex: 0, type: EMBED_CS },
+			{ startIndex: 0, type: 'metatag.cs' },
 			{ startIndex: 2, type: '' },
 			{ startIndex: 3, type: 'keyword.cs' },
 			{ startIndex: 6, type: '' },
 			{ startIndex: 7, type: 'identifier.cs' },
 			{ startIndex: 8, type: 'delimiter.cs' },
 			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 11, type: htmlTokenTypes.getTag('b') },
-			{ startIndex: 12, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 10, type: 'delimiter.html' },
+			{ startIndex: 11, type: 'tag.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
 			{ startIndex: 13, type: 'identifier.cs' },
-			{ startIndex: 14, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 16, type: htmlTokenTypes.getTag('b') },
-			{ startIndex: 17, type: htmlTokenTypes.DELIM_END },
+			{ startIndex: 14, type: 'delimiter.html' },
+			{ startIndex: 16, type: 'tag.html' },
+			{ startIndex: 17, type: 'delimiter.html' },
 			{ startIndex: 18, type: '' },
-			{ startIndex: 19, type: EMBED_CS }
+			{ startIndex: 19, type: 'metatag.cs' }
 		]
 	}],
 
@@ -39,7 +36,7 @@ testTokenization('razor', [
 	[{
 		line: '@{ var x; @* comment *@ x= 0; }',
 		tokens: [
-			{ startIndex: 0, type: EMBED_CS },
+			{ startIndex: 0, type: 'metatag.cs' },
 			{ startIndex: 2, type: '' },
 			{ startIndex: 3, type: 'keyword.cs' },
 			{ startIndex: 6, type: '' },
@@ -54,7 +51,7 @@ testTokenization('razor', [
 			{ startIndex: 27, type: 'number.cs' },
 			{ startIndex: 28, type: 'delimiter.cs' },
 			{ startIndex: 29, type: '' },
-			{ startIndex: 30, type: EMBED_CS }
+			{ startIndex: 30, type: 'metatag.cs' }
 		]
 	}],
 
@@ -62,7 +59,7 @@ testTokenization('razor', [
 	[{
 		line: '@{ var total = 0; }',
 		tokens: [
-			{ startIndex: 0, type: EMBED_CS },
+			{ startIndex: 0, type: 'metatag.cs' },
 			{ startIndex: 2, type: '' },
 			{ startIndex: 3, type: 'keyword.cs' },
 			{ startIndex: 6, type: '' },
@@ -73,19 +70,19 @@ testTokenization('razor', [
 			{ startIndex: 15, type: 'number.cs' },
 			{ startIndex: 16, type: 'delimiter.cs' },
 			{ startIndex: 17, type: '' },
-			{ startIndex: 18, type: EMBED_CS }
+			{ startIndex: 18, type: 'metatag.cs' }
 		]
 	}],
 
 	// [{
 	// line: '@if(true){ var total = 0; }',
 	// tokens: [
-	// 	{ startIndex: 0, type: EMBED_CS },
+	// 	{ startIndex: 0, type: 'metatag.cs' },
 	// 	{ startIndex: 1, type: 'keyword.cs' },
 	// 	{ startIndex: 3, type: 'punctuation.parenthesis.cs' },
 	// 	{ startIndex: 4, type: 'keyword.cs' },
 	// 	{ startIndex: 8, type: 'punctuation.parenthesis.cs' },
-	// 	{ startIndex: 9, type: EMBED_CS },
+	// 	{ startIndex: 9, type: 'metatag.cs' },
 	// 	{ startIndex: 10, type: '' },
 	// 	{ startIndex: 11, type: 'keyword.cs' },
 	// 	{ startIndex: 14, type: '' },
@@ -96,7 +93,7 @@ testTokenization('razor', [
 	// 	{ startIndex: 23, type: 'number.cs' },
 	// 	{ startIndex: 24, type: 'delimiter.cs' },
 	// 	{ startIndex: 25, type: '' },
-	// 	{ startIndex: 26, type: EMBED_CS }
+	// 	{ startIndex: 26, type: 'metatag.cs' }
 	// ]}],
 
 	// Expressions - csharp expressions in html
@@ -104,11 +101,11 @@ testTokenization('razor', [
 		line: 'test@xyz<br>',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 4, type: EMBED_CS },
+			{ startIndex: 4, type: 'metatag.cs' },
 			{ startIndex: 5, type: 'identifier.cs' },
-			{ startIndex: 8, type: htmlTokenTypes.DELIM_START },
-			{ startIndex: 9, type: htmlTokenTypes.getTag('br') },
-			{ startIndex: 11, type: htmlTokenTypes.DELIM_END }
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'tag.html' },
+			{ startIndex: 11, type: 'delimiter.html' }
 		]
 	}],
 
@@ -116,7 +113,7 @@ testTokenization('razor', [
 		line: 'test@xyz',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 4, type: EMBED_CS },
+			{ startIndex: 4, type: 'metatag.cs' },
 			{ startIndex: 5, type: 'identifier.cs' }
 		]
 	}],
@@ -125,7 +122,7 @@ testTokenization('razor', [
 		line: 'test @ xyz',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 5, type: EMBED_CS },
+			{ startIndex: 5, type: 'metatag.cs' },
 			{ startIndex: 6, type: 'identifier.cs' }
 		]
 	}],
@@ -134,9 +131,9 @@ testTokenization('razor', [
 		line: 'test @(foo) xyz',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 5, type: EMBED_CS },
+			{ startIndex: 5, type: 'metatag.cs' },
 			{ startIndex: 7, type: 'identifier.cs' },
-			{ startIndex: 10, type: EMBED_CS },
+			{ startIndex: 10, type: 'metatag.cs' },
 			{ startIndex: 11, type: '' }
 		]
 	}],
@@ -145,12 +142,12 @@ testTokenization('razor', [
 		line: 'test @(foo(\")\")) xyz',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 5, type: EMBED_CS },
+			{ startIndex: 5, type: 'metatag.cs' },
 			{ startIndex: 7, type: 'identifier.cs' },
 			{ startIndex: 10, type: 'delimiter.parenthesis.cs' },
 			{ startIndex: 11, type: 'string.cs' },
 			{ startIndex: 14, type: 'delimiter.parenthesis.cs' },
-			{ startIndex: 15, type: EMBED_CS },
+			{ startIndex: 15, type: 'metatag.cs' },
 			{ startIndex: 16, type: '' }
 		]
 	}],

test/testRunner.ts

@@ -7,7 +7,7 @@ import {loadLanguage} from '../src/monaco.contribution';
 import * as assert from 'assert';
 
 // Allow for running under nodejs/requirejs in tests
-var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+const _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
 
 export interface IRelaxedToken {
 	startIndex: number;