Small tweaks to use classic-style tokens

src/handlebars.ts

@@ -215,14 +215,14 @@ export var language = <ILanguage> {
 
 
 		handlebarsInSimpleState: [
-			[/\{\{\{?/, 'punctuation.handlebars'],
-			[/\}\}\}?/, { token: 'punctuation.handlebars', switchTo: '@$S2.$S3' }],
+			[/\{\{\{?/, 'delimiter.handlebars'],
+			[/\}\}\}?/, { token: 'delimiter.handlebars', switchTo: '@$S2.$S3' }],
 			{ include: 'handlebarsRoot' }
 		],
 
 		handlebarsInEmbeddedState: [
-			[/\{\{\{?/, 'punctuation.handlebars'],
-			[/\}\}\}?/, { token: 'punctuation.handlebars', switchTo: '@$S2.$S3', nextEmbedded: '$S3' }],
+			[/\{\{\{?/, 'delimiter.handlebars'],
+			[/\}\}\}?/, { token: 'delimiter.handlebars', switchTo: '@$S2.$S3', nextEmbedded: '$S3' }],
 			{ include: 'handlebarsRoot' }
 		],
 

src/markdown.ts

@@ -8,24 +8,18 @@
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
-const TOKEN_HEADER_LEAD = 'entity.name.tag';
-const TOKEN_HEADER = 'entity.name.tag';
-const TOKEN_EXT_HEADER = 'entity.other.attribute-name';
+const TOKEN_HEADER_LEAD = 'keyword';
+const TOKEN_HEADER = 'keyword';
+const TOKEN_EXT_HEADER = 'keyword';
 const TOKEN_SEPARATOR = 'meta.separator';
 const TOKEN_QUOTE = 'comment';
 const TOKEN_LIST = 'keyword';
 const TOKEN_BLOCK = 'string';
 const TOKEN_BLOCK_CODE = 'variable.source';
 
-const DELIM_END = 'punctuation.definition.meta.tag.end.html';
-const DELIM_START = 'punctuation.definition.meta.tag.begin.html';
 const DELIM_ASSIGN = 'meta.tag.assign.html';
-const ATTRIB_NAME = 'entity.other.attribute-name.html';
+const ATTRIB_NAME = 'attribute.name.html';
 const ATTRIB_VALUE = 'string.html';
-const COMMENT = 'comment.html.content';
-const DELIM_COMMENT = 'comment.html';
-const DOCTYPE = 'entity.other.attribute-name.html';
-const DELIM_DOCTYPE = 'entity.name.tag.html';
 
 const TAG_PREFIX = 'entity.name.tag.tag-';
 

src/swift.ts

@@ -69,8 +69,8 @@ export var language = <ILanguage> {
 			[/[{}()\[\]]/, '@brackets'],
 			[/[<>](?!@symbols)/, '@brackets'],
 			[/[.]/, 'delimiter'],
-			[/@operators/, 'keyword.operator'],
-			[/@symbols/, 'keyword.operator']
+			[/@operators/, 'operator'],
+			[/@symbols/, 'operator']
 		],
 
 
@@ -106,7 +106,7 @@ export var language = <ILanguage> {
 		],
 
 		stringlit: [
-			[ /\\\(/, { token: 'keyword.operator', bracket: '@open', next: '@interpolatedexpression' } ],
+			[ /\\\(/, { token: 'operator', bracket: '@open', next: '@interpolatedexpression' } ],
 			[ /@escapes/, 'string' ],
 			[ /\\./, 'string.escape.invalid' ],
 			[ /"/, { token: 'string.quote', bracket: '@close', next: '@pop' } ],
@@ -114,26 +114,26 @@ export var language = <ILanguage> {
 		],
 
 		interpolatedexpression: [
-			[ /\(/, { token: 'keyword.operator', bracket: '@open', next: '@interpolatedexpression' } ],
-			[ /\)/, { token: 'keyword.operator', bracket: '@close', next: '@pop' } ],
+			[ /\(/, { token: 'operator', bracket: '@open', next: '@interpolatedexpression' } ],
+			[ /\)/, { token: 'operator', bracket: '@close', next: '@pop' } ],
 			{ include: '@literal' },
 			{ include: '@keyword' },
 			{ include: '@symbol' }
 		],
 
 		keyword: [
-			[ /`/, { token: 'keyword.operator', bracket: '@open', next: '@escapedkeyword' } ],
+			[ /`/, { token: 'operator', bracket: '@open', next: '@escapedkeyword' } ],
 			[ /@identifier/, { cases: { '@keywords': 'keyword', '[A-Z][\a-zA-Z0-9$]*': 'type.identifier', '@default': 'identifier' } }]
 		],
 
 		escapedkeyword: [
-			[ /`/, { token: 'keyword.operator', bracket: '@close', next: '@pop' } ],
+			[ /`/, { token: 'operator', bracket: '@close', next: '@pop' } ],
 			[ /./, 'identifier' ]
 		],
 
 //		symbol: [
-//			[ /@symbols/, 'keyword.operator' ],
-//			[ /@operators/, 'keyword.operator' ]
+//			[ /@symbols/, 'operator' ],
+//			[ /@operators/, 'operator' ]
 //		],
 
 		invokedmethod: [

test/handlebars.test.ts

@@ -18,8 +18,8 @@ function getTag(name: string) {
 }
 
 const handlebarsTokenTypes = {
-	EMBED: 'punctuation.handlebars',
-	EMBED_UNESCAPED: 'punctuation.handlebars',
+	EMBED: 'delimiter.handlebars',
+	EMBED_UNESCAPED: 'delimiter.handlebars',
 	KEYWORD: 'keyword.helper.handlebars',
 	VARIABLE: 'variable.parameter.handlebars',
 }

test/markdown.test.ts

@@ -12,7 +12,7 @@ testTokenization('markdown', [
     [{
         line: '# Some header',
         tokens: [
-            { startIndex: 0, type: 'entity.name.tag.md' }
+            { startIndex: 0, type: 'keyword.md' }
         ]
     }],
 

test/swift.test.ts

@@ -22,13 +22,13 @@ testTokenization('swift', [
 		{ startIndex: 0, type: 'keyword.swift' } /* 'class' */,
 		{ startIndex: 5, type: '' },
 		{ startIndex: 6, type: 'type.identifier.swift' } /* 'App' */,
-		{ startIndex: 9, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 9, type: 'operator.swift' } /* ':' */,
 		{ startIndex: 10, type: '' },
 		{ startIndex: 11, type: 'type.identifier.swift' } /* 'UI' */,
-		{ startIndex: 13, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 13, type: 'operator.swift' } /* ',' */,
 		{ startIndex: 14, type: '' },
 		{ startIndex: 15, type: 'type.identifier.swift' } /* 'UIApp' */,
-		{ startIndex: 20, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 20, type: 'operator.swift' } /* ',' */,
 		{ startIndex: 21, type: '' },
 		{ startIndex: 22, type: 'type.identifier.swift' } /* 'UIView' */,
 		{ startIndex: 28, type: '' },
@@ -42,10 +42,10 @@ testTokenization('swift', [
 		{ startIndex: 4, type: 'keyword.swift' } /* 'var' */,
 		{ startIndex: 7, type: '' },
 		{ startIndex: 8, type: 'identifier.swift' } /* 'window' */,
-		{ startIndex: 14, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 14, type: 'operator.swift' } /* ':' */,
 		{ startIndex: 15, type: '' },
 		{ startIndex: 16, type: 'type.identifier.swift' } /* 'UIWindow' */,
-		{ startIndex: 24, type: 'keyword.operator.swift' } /* '?' */
+		{ startIndex: 24, type: 'operator.swift' } /* '?' */
 	]}],
 	//Comment
 	[{
@@ -65,7 +65,7 @@ testTokenization('swift', [
 		{ startIndex: 32, type: '' },
 		{ startIndex: 33, type: 'identifier.swift' }	/* 'y' */,
 		{ startIndex: 34, type: '' },
-		{ startIndex: 35, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 35, type: 'operator.swift' } /* '=' */,
 		{ startIndex: 36, type: '' },
 		{ startIndex: 37, type: 'number.binary.swift' } /* '0b10' */
 	]}],
@@ -81,13 +81,13 @@ testTokenization('swift', [
 		{ startIndex: 16, type: 'identifier.swift' } /* 'app' */,
 		{ startIndex: 19, type: 'delimiter.parenthesis.swift' } /* '(' */,
 		{ startIndex: 20, type: 'identifier.swift' }/* 'app' */,
-		{ startIndex: 23, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 23, type: 'operator.swift' } /* ':' */,
 		{ startIndex: 24, type: '' },
 		{ startIndex: 25, type: 'type.identifier.swift' } /* 'App' */,
-		{ startIndex: 28, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 28, type: 'operator.swift' } /* ',' */,
 		{ startIndex: 29, type: '' },
 		{ startIndex: 30, type: 'identifier.swift' } /* 'opts' */,
-		{ startIndex: 34, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 34, type: 'operator.swift' } /* ':' */,
 	]}],
 	// Method signature Continued
 	[{
@@ -96,14 +96,14 @@ testTokenization('swift', [
 		{ startIndex: 0, type: '' },
 		{ startIndex: 8, type: 'delimiter.square.swift' } /* '[' */,
 		{ startIndex: 9, type: 'type.identifier.swift' } /* 'NSObject' */,
-		{ startIndex: 17, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 17, type: 'operator.swift' } /* ':' */,
 		{ startIndex: 18, type: '' },
 		{ startIndex: 19, type: 'type.identifier.swift' } /* 'AnyObject' */,
 		{ startIndex: 28, type: 'delimiter.square.swift' } /* ']' */,
-		{ startIndex: 29, type: 'keyword.operator.swift' } /* '?' */,
+		{ startIndex: 29, type: 'operator.swift' } /* '?' */,
 		{ startIndex: 30, type: 'delimiter.parenthesis.swift' } /* ')' */,
 		{ startIndex: 31, type: '' },
-		{ startIndex: 32, type: 'keyword.operator.swift' } /* '->' */,
+		{ startIndex: 32, type: 'operator.swift' } /* '->' */,
 		{ startIndex: 34, type: '' },
 		{ startIndex: 35, type: 'type.identifier.swift' } /* 'Bool' */,
 		{ startIndex: 39, type: '' },
@@ -116,11 +116,11 @@ testTokenization('swift', [
 		{ startIndex: 0, type: '' },
 		{ startIndex: 8, type: 'keyword.swift' } /* 'var' */,
 		{ startIndex: 11, type: '' },
-		{ startIndex: 12, type: 'keyword.operator.swift' } /* '`' */,
+		{ startIndex: 12, type: 'operator.swift' } /* '`' */,
 		{ startIndex: 13, type: 'identifier.swift' } /* 'String' */,
-		{ startIndex: 19, type: 'keyword.operator.swift' } /* '`' */,
+		{ startIndex: 19, type: 'operator.swift' } /* '`' */,
 		{ startIndex: 20, type: '' },
-		{ startIndex: 21, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 21, type: 'operator.swift' } /* '=' */,
 		{ startIndex: 22, type: '' },
 		{ startIndex: 23, type: 'string.quote.swift' } /* '"' */,
 		{ startIndex: 24, type: 'string.swift' } /* 'String w/ \\"escape\\""' */,
@@ -135,23 +135,23 @@ testTokenization('swift', [
 		{ startIndex: 11, type: '' },
 		{ startIndex: 12, type: 'identifier.swift' } /* 'message' */,
 		{ startIndex: 19, type: '' },
-		{ startIndex: 20, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 20, type: 'operator.swift' } /* '=' */,
 		{ startIndex: 21, type: '' },
 		{ startIndex: 22, type: 'string.quote.swift' } /* '"' */,
-		{ startIndex: 23, type: 'keyword.operator.swift' } /* '\(' */,
+		{ startIndex: 23, type: 'operator.swift' } /* '\(' */,
 		{ startIndex: 25, type: 'identifier.swift' },
-		{ startIndex: 26, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 26, type: 'operator.swift' } /* ')' */,
 		{ startIndex: 27, type: 'string.swift' } /* ' times 2.5 is ' */,
-		{ startIndex: 41, type: 'keyword.operator.swift' } /* '\(' */,
+		{ startIndex: 41, type: 'operator.swift' } /* '\(' */,
 		{ startIndex: 43, type: 'type.identifier.swift' } /* 'Double' */,
-		{ startIndex: 49, type: 'keyword.operator.swift' } /* '(' */,
+		{ startIndex: 49, type: 'operator.swift' } /* '(' */,
 		{ startIndex: 50, type: 'number.swift' } /* '25' */,
-		{ startIndex: 52, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 52, type: 'operator.swift' } /* ')' */,
 		{ startIndex: 53, type: '' },
-		{ startIndex: 54, type: 'keyword.operator.swift' } /* '*' */,
+		{ startIndex: 54, type: 'operator.swift' } /* '*' */,
 		{ startIndex: 55, type: '' },
 		{ startIndex: 56, type: 'number.float.swift' } /* '2.5' */,
-		{ startIndex: 59, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 59, type: 'operator.swift' } /* ')' */,
 		{ startIndex: 60, type: 'string.quote.swift' } /* '"' */
 	]}],
 	// Method invocation/property accessor.
@@ -163,17 +163,17 @@ testTokenization('swift', [
 		{ startIndex: 11, type: '' },
 		{ startIndex: 12, type: 'identifier.swift' } /* 'view' */,
 		{ startIndex: 16, type: '' },
-		{ startIndex: 17, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 17, type: 'operator.swift' } /* '=' */,
 		{ startIndex: 18, type: '' },
 		{ startIndex: 19, type: 'keyword.swift' } /* 'self' */,
 		{ startIndex: 23, type: 'delimeter.swift' } /* '.' */,
 		{ startIndex: 24, type: 'type.identifier.swift' } /* 'window' */,
-		{ startIndex: 30, type: 'keyword.operator.swift' } /* '!' */,
+		{ startIndex: 30, type: 'operator.swift' } /* '!' */,
 		{ startIndex: 31, type: 'delimeter.swift' } /* '.' */,
 		{ startIndex: 32, type: 'type.identifier.swift' } /* 'contr' */,
 		{ startIndex: 37, type: '' },
 		{ startIndex: 38, type: 'keyword.swift' } /* 'as' */,
-		{ startIndex: 40, type: 'keyword.operator.swift' } /* '!' */,
+		{ startIndex: 40, type: 'operator.swift' } /* '!' */,
 		{ startIndex: 41, type: '' },
 		{ startIndex: 42, type: 'type.identifier.swift' } /* 'UIView' */
 	]}]