adopt latest editor

src/markdown.ts

@@ -75,33 +75,28 @@ export var language = <ILanguage>{
             [/^(\t|[ ]{4})[^ ].*$/, TOKEN_BLOCK],
 
             // code block (3 tilde)
-            [/^\s*~{3}\s*((?:\w|[\/\-#])+)?\s*$/, { token: TOKEN_BLOCK, next: '@codeblock' }],
+            [/^\s*~~~\s*((?:\w|[\/\-#])+)?\s*$/, { token: TOKEN_BLOCK, next: '@codeblock' }],
 
             // github style code blocks (with backticks and language)
             [/^\s*```\s*((?:\w|[\/\-#])+)\s*$/, { token: TOKEN_BLOCK, next: '@codeblockgh', nextEmbedded: '$1' }],
 
             // github style code blocks (with backticks but no language)
-            [/^\s*`{3}\s*$/, { token: TOKEN_BLOCK, next: '@codeblock' }],
+            [/^\s*```\s*$/, { token: TOKEN_BLOCK, next: '@codeblock' }],
 
             // markup within lines
             { include: '@linecontent' },
         ],
 
         codeblock: [
-            [/^\s*~{3}\s*$/, { token: TOKEN_BLOCK, next: '@pop' }],
-            [/^\s*`{3}\s*$/, { token: TOKEN_BLOCK, next: '@pop' }],
+            [/^\s*~~~\s*$/, { token: TOKEN_BLOCK, next: '@pop' }],
+            [/^\s*```\s*$/, { token: TOKEN_BLOCK, next: '@pop' }],
             [/.*$/, TOKEN_BLOCK_CODE],
         ],
 
         // github style code blocks
         codeblockgh: [
-            [/```\s*$/, { token: '@rematch', switchTo: '@codeblockghend', nextEmbedded: '@pop' }],
-            [/[^`]*$/, TOKEN_BLOCK_CODE],
-        ],
-
-        codeblockghend: [
-            [/\s*```/, { token: TOKEN_BLOCK_CODE, next: '@pop' }],
-            [/./, '@rematch', '@pop'],
+            [/```\s*$/, { token: TOKEN_BLOCK_CODE, next: '@pop', nextEmbedded: '@pop' }],
+            [/[^`]+/, TOKEN_BLOCK_CODE],
         ],
 
         linecontent: [
@@ -166,11 +161,11 @@ export var language = <ILanguage>{
             [/\/>/, getTag('$S2'), '@pop'],
             [/>/, {
                 cases: {
-                    '$S2==style': { token: getTag('$S2'), switchTo: '@embedded.$S2', nextEmbedded: 'text/css' },
+                    '$S2==style': { token: getTag('$S2'), switchTo: 'embeddedStyle', nextEmbedded: 'text/css' },
                     '$S2==script': {
                         cases: {
-                            '$S3': { token: getTag('$S2'), switchTo: '@embedded.$S2', nextEmbedded: '$S3' },
-                            '@default': { token: getTag('$S2'), switchTo: '@embedded.$S2', nextEmbedded: 'text/javascript' }
+                            '$S3': { token: getTag('$S2'), switchTo: 'embeddedScript', nextEmbedded: '$S3' },
+                            '@default': { token: getTag('$S2'), switchTo: 'embeddedScript', nextEmbedded: 'text/javascript' }
                         }
                     },
                     '@default': { token: getTag('$S2'), next: '@pop' }
@@ -178,32 +173,16 @@ export var language = <ILanguage>{
             }],
         ],
 
-        embedded: [
-            [/[^"'<]+/, ''],
-            [/<\/(\w+)\s*>/, {
-                cases: {
-                    '$1==$S2': { token: '@rematch', next: '@pop', nextEmbedded: '@pop' },
-                    '@default': ''
-                }
-            }],
-            [/"([^"\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
-            [/'([^'\\]|\\.)*$/, 'string.invalid'],  // non-teminated string
-            [/"/, 'string', '@string."'],
-            [/'/, 'string', '@string.\''],
+        embeddedStyle: [
+            [/[^<]+/, ''],
+            [/<\/style\s*>/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }],
             [/</, '']
         ],
 
-        // scan embedded strings in javascript or css
-        string: [
-            [/[^\\"']+/, 'string'],
-            [/@jsescapes/, 'string.escape'],
-            [/\\./, 'string.escape.invalid'],
-            [/["']/, {
-                cases: {
-                    '$#==$S2': { token: 'string', next: '@pop' },
-                    '@default': 'string'
-                }
-            }]
-        ]
+        embeddedScript: [
+            [/[^<]+/, ''],
+            [/<\/script\s*>/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }],
+            [/</, '']
+        ],
     }
 };

test/testRunner.ts

@@ -46,53 +46,19 @@ function runTests(languageId:string, tests:ITestItem[][]): void {
 
 function runTest(languageId:string, test:ITestItem[]): void {
 
-	interface LineToken {
-		startOffset: number;
-		endOffset: number;
-		type: string;
-		modeId: string;
-		hasPrev: boolean;
-		hasNext: boolean;
-
-		prev(): LineToken;
-		next(): LineToken;
-	}
-
-	interface LineTokens {
-		// modeTransitions:ModeTransition[];
-
-		getTokenCount(): number;
-		getTokenStartOffset(tokenIndex:number): number;
-		getTokenType(tokenIndex:number): string;
-		getTokenEndOffset(tokenIndex:number): number;
-		equals(other:LineTokens): boolean;
-		findTokenIndexAtOffset(offset:number): number;
-		findTokenAtOffset(offset:number): LineToken;
-		firstToken(): LineToken;
-		lastToken(): LineToken;
-		// inflate(): ViewLineToken[];
-		// sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[];
-	}
-
 	let text = test.map(t => t.line).join('\n');
-	let model = _monaco.editor.createModel(text, languageId);
-
-	for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber <= lineCount; lineNumber++) {
-		let actual: IRelaxedToken[] = [];
-		let lineTokens:LineTokens = (<any>model).getLineTokens(lineNumber);
-		let count = lineTokens.getTokenCount();
-		for (let i = 0; i < count; i++) {
-			actual.push({
-				startIndex: lineTokens.getTokenStartOffset(i),
-				type: lineTokens.getTokenType(i)
-			});
-		}
-
-		let expected = test[lineNumber - 1].tokens;
-		// console.log(`actual: ${JSON.stringify(actual)}`);
-		// console.log(`expected: ${JSON.stringify(expected)}`);
-		assert.deepEqual(actual, expected, 'TOKENIZING ' + text);
-	}
+	let actualTokens = _monaco.editor.tokenize(text, languageId);
+	let actual = actualTokens.map((lineTokens, index) => {
+		return {
+			line: test[index].line,
+			tokens: lineTokens.map((t) => {
+				return {
+					startIndex: t.offset,
+					type: t.type
+				};
+			})
+		};
+	});
 
-	model.dispose();
+	assert.deepEqual(actual, test);
 }