Merge pull request #94 from skacurt/vb-string

[vb] fix string literals

src/vb/vb.test.ts

@@ -289,18 +289,95 @@ testTokenization('vb', [
 			{ startIndex: 8, type: '' },
 			{ startIndex: 9, type: 'delimiter.vb' },
 			{ startIndex: 10, type: '' },
-			{ startIndex: 11, type: 'string.vb' }
+			{ startIndex: 11, type: 'string.quote.vb' },
+			{ startIndex: 12, type: 'string.vb' },
+			{ startIndex: 18, type: 'string.quote.vb' }
 		]
 	}],
 
 	[{
 		line: '"use strict";',
 		tokens: [
-			{ startIndex: 0, type: 'string.vb' },
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 11, type: 'string.quote.vb' },
 			{ startIndex: 12, type: 'delimiter.vb' }
 		]
 	}],
 
+	[{
+		line: '"a""b"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 2, type: 'string.escape.vb' },
+			{ startIndex: 4, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '"a““b"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 2, type: 'string.escape.vb' },
+			{ startIndex: 4, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '"a””b"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 2, type: 'string.escape.vb' },
+			{ startIndex: 4, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
+		]
+	}],
+
+	[{
+		line: '"mixed quotes 1“',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '"mixed quotes 2”',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '“mixed quotes 3"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '“mixed quotes 4”',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '”mixed quotes 5"',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}, {
+		line: '”mixed quotes 6“',
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.vb' },
+			{ startIndex: 1, type: 'string.vb' },
+			{ startIndex: 15, type: 'string.quote.vb' }
+		]
+	}],
+
 	// Tags
 	[{
 		line: 'Public Sub ToString()',
@@ -445,12 +522,14 @@ testTokenization('vb', [
 			{ startIndex: 5, type: '' },
 			{ startIndex: 6, type: 'delimiter.vb' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'string.vb' }
+			{ startIndex: 8, type: 'string.quote.vb' },
+			{ startIndex: 9, type: 'string.vb' }
 		]
 	}, {
 		line: 'world"',
 		tokens: [
 			{ startIndex: 0, type: 'string.vb' },
+			{ startIndex: 5, type: 'string.quote.vb' }
 		]
 	}],
 

src/vb/vb.ts

@@ -124,7 +124,6 @@ export const language = <ILanguage>{
 
 	// we include these common regular expressions
 	symbols: /[=><!~?;\.,:&|+\-*\/\^%]+/,
-	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
 	integersuffix: /U?[DI%L&S@]?/,
 	floatsuffix: /[R#F!]?/,
 
@@ -169,7 +168,7 @@ export const language = <ILanguage>{
 			[/@symbols/, 'delimiter'],
 
 			// strings
-			[/"/, 'string', '@string'],
+			[/["\u201c\u201d]/, { token: 'string.quote', next: '@string' }],
 
 		],
 
@@ -179,10 +178,9 @@ export const language = <ILanguage>{
 		],
 
 		string: [
-			[/[^\\"]+/, 'string'],
-			[/@escapes/, 'string.escape'],
-			[/\\./, 'string.escape.invalid'],
-			[/"C?/, 'string', '@pop']
+			[/[^"\u201c\u201d]+/, 'string'],
+			[/["\u201c\u201d]{2}/, 'string.escape'],
+			[/["\u201c\u201d]C?/, { token: 'string.quote', next: '@pop' }]
 		],
 	},
 };