Merge pull request #80 from nrayburn-tech/master

Add support for multiline comments in Swift

src/swift/swift.test.ts

@@ -21,18 +21,18 @@ testTokenization('swift', [
 		line: 'class App: UI, UIApp, UIView {',
 		tokens: [
 			{ startIndex: 0, type: 'keyword.swift' } /* 'class' */,
-			{ startIndex: 5, type: '' },
+			{ startIndex: 5, type: 'white.swift' },
 			{ startIndex: 6, type: 'type.identifier.swift' } /* 'App' */,
 			{ startIndex: 9, type: 'operator.swift' } /* ':' */,
-			{ startIndex: 10, type: '' },
+			{ startIndex: 10, type: 'white.swift' },
 			{ startIndex: 11, type: 'type.identifier.swift' } /* 'UI' */,
 			{ startIndex: 13, type: 'operator.swift' } /* ',' */,
-			{ startIndex: 14, type: '' },
+			{ startIndex: 14, type: 'white.swift' },
 			{ startIndex: 15, type: 'type.identifier.swift' } /* 'UIApp' */,
 			{ startIndex: 20, type: 'operator.swift' } /* ',' */,
-			{ startIndex: 21, type: '' },
+			{ startIndex: 21, type: 'white.swift' },
 			{ startIndex: 22, type: 'type.identifier.swift' } /* 'UIView' */,
-			{ startIndex: 28, type: '' },
+			{ startIndex: 28, type: 'white.swift' },
 			{ startIndex: 29, type: 'delimiter.curly.swift' } /* '{' */
 		]
 	}],
@@ -40,12 +40,12 @@ testTokenization('swift', [
 	[{
 		line: '    var window: UIWindow?',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 4, type: 'keyword.swift' } /* 'var' */,
-			{ startIndex: 7, type: '' },
+			{ startIndex: 7, type: 'white.swift' },
 			{ startIndex: 8, type: 'identifier.swift' } /* 'window' */,
 			{ startIndex: 14, type: 'operator.swift' } /* ':' */,
-			{ startIndex: 15, type: '' },
+			{ startIndex: 15, type: 'white.swift' },
 			{ startIndex: 16, type: 'type.identifier.swift' } /* 'UIWindow' */,
 			{ startIndex: 24, type: 'operator.swift' } /* '?' */
 		]
@@ -54,7 +54,7 @@ testTokenization('swift', [
 	[{
 		line: '    // Comment',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 4, type: 'comment.swift' } /* '// Comment' */
 		]
 	}],
@@ -62,15 +62,15 @@ testTokenization('swift', [
 	[{
 		line: '    /* Comment //Embedded */ var y = 0b10',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 4, type: 'comment.swift' }, // /* '/* Comment //Embedded */' */,
-			{ startIndex: 28, type: '' },
+			{ startIndex: 28, type: 'white.swift' },
 			{ startIndex: 29, type: 'keyword.swift' } /* 'var' */,
-			{ startIndex: 32, type: '' },
+			{ startIndex: 32, type: 'white.swift' },
 			{ startIndex: 33, type: 'identifier.swift' }	/* 'y' */,
-			{ startIndex: 34, type: '' },
+			{ startIndex: 34, type: 'white.swift' },
 			{ startIndex: 35, type: 'operator.swift' } /* '=' */,
-			{ startIndex: 36, type: '' },
+			{ startIndex: 36, type: 'white.swift' },
 			{ startIndex: 37, type: 'number.binary.swift' } /* '0b10' */
 		]
 	}],
@@ -78,19 +78,19 @@ testTokenization('swift', [
 	[{
 		line: '    public func app(app: App, opts:',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 4, type: 'keyword.swift' } /* 'public' */,
-			{ startIndex: 10, type: '' },
+			{ startIndex: 10, type: 'white.swift' },
 			{ startIndex: 11, type: 'keyword.swift' } /* 'func' */,
-			{ startIndex: 15, type: '' },
+			{ startIndex: 15, type: 'white.swift' },
 			{ startIndex: 16, type: 'identifier.swift' } /* 'app' */,
 			{ startIndex: 19, type: 'delimiter.parenthesis.swift' } /* '(' */,
 			{ startIndex: 20, type: 'identifier.swift' }/* 'app' */,
 			{ startIndex: 23, type: 'operator.swift' } /* ':' */,
-			{ startIndex: 24, type: '' },
+			{ startIndex: 24, type: 'white.swift' },
 			{ startIndex: 25, type: 'type.identifier.swift' } /* 'App' */,
 			{ startIndex: 28, type: 'operator.swift' } /* ',' */,
-			{ startIndex: 29, type: '' },
+			{ startIndex: 29, type: 'white.swift' },
 			{ startIndex: 30, type: 'identifier.swift' } /* 'opts' */,
 			{ startIndex: 34, type: 'operator.swift' } /* ':' */,
 		]
@@ -99,20 +99,20 @@ testTokenization('swift', [
 	[{
 		line: '        [NSObject: AnyObject]?) -> Bool {',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 8, type: 'delimiter.square.swift' } /* '[' */,
 			{ startIndex: 9, type: 'type.identifier.swift' } /* 'NSObject' */,
 			{ startIndex: 17, type: 'operator.swift' } /* ':' */,
-			{ startIndex: 18, type: '' },
+			{ startIndex: 18, type: 'white.swift' },
 			{ startIndex: 19, type: 'type.identifier.swift' } /* 'AnyObject' */,
 			{ startIndex: 28, type: 'delimiter.square.swift' } /* ']' */,
 			{ startIndex: 29, type: 'operator.swift' } /* '?' */,
 			{ startIndex: 30, type: 'delimiter.parenthesis.swift' } /* ')' */,
-			{ startIndex: 31, type: '' },
+			{ startIndex: 31, type: 'white.swift' },
 			{ startIndex: 32, type: 'operator.swift' } /* '->' */,
-			{ startIndex: 34, type: '' },
+			{ startIndex: 34, type: 'white.swift' },
 			{ startIndex: 35, type: 'type.identifier.swift' } /* 'Bool' */,
-			{ startIndex: 39, type: '' },
+			{ startIndex: 39, type: 'white.swift' },
 			{ startIndex: 40, type: 'delimiter.curly.swift' } /* '{' */
 		]
 	}],
@@ -120,15 +120,15 @@ testTokenization('swift', [
 	[{
 		line: '        var `String` = "String w/ \\"escape\\""',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 8, type: 'keyword.swift' } /* 'var' */,
-			{ startIndex: 11, type: '' },
+			{ startIndex: 11, type: 'white.swift' },
 			{ startIndex: 12, type: 'operator.swift' } /* '`' */,
 			{ startIndex: 13, type: 'identifier.swift' } /* 'String' */,
 			{ startIndex: 19, type: 'operator.swift' } /* '`' */,
-			{ startIndex: 20, type: '' },
+			{ startIndex: 20, type: 'white.swift' },
 			{ startIndex: 21, type: 'operator.swift' } /* '=' */,
-			{ startIndex: 22, type: '' },
+			{ startIndex: 22, type: 'white.swift' },
 			{ startIndex: 23, type: 'string.quote.swift' } /* '"' */,
 			{ startIndex: 24, type: 'string.swift' } /* 'String w/ \\"escape\\""' */,
 			{ startIndex: 44, type: 'string.quote.swift' } /* '"' */,
@@ -138,13 +138,13 @@ testTokenization('swift', [
 	[{
 		line: '        let message = "\\(y) times 2.5 is \\(Double(25) * 2.5)"',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 8, type: 'keyword.swift' } /* 'let' */,
-			{ startIndex: 11, type: '' },
+			{ startIndex: 11, type: 'white.swift' },
 			{ startIndex: 12, type: 'identifier.swift' } /* 'message' */,
-			{ startIndex: 19, type: '' },
+			{ startIndex: 19, type: 'white.swift' },
 			{ startIndex: 20, type: 'operator.swift' } /* '=' */,
-			{ startIndex: 21, type: '' },
+			{ startIndex: 21, type: 'white.swift' },
 			{ startIndex: 22, type: 'string.quote.swift' } /* '"' */,
 			{ startIndex: 23, type: 'operator.swift' } /* '\(' */,
 			{ startIndex: 25, type: 'identifier.swift' },
@@ -163,27 +163,41 @@ testTokenization('swift', [
 			{ startIndex: 60, type: 'string.quote.swift' } /* '"' */
 		]
 	}],
+	// Multiline string
+	[{
+		line: '"""test"test', // Separate new lines into separate objects within this same array.
+		tokens: [
+			{ startIndex: 0, type: 'string.quote.swift' }, /* '"""' */
+			{ startIndex: 3, type: 'string.swift' } /* test"test */
+		]
+	}, {
+		line: ' keepsgoing"""',
+		tokens: [
+			{ startIndex: 0, type: 'string.swift' }, /* ' keepsgoing' */
+			{ startIndex: 11, type: 'string.quote.swift' } /* '"""' */
+		]
+	}],
 	// Method invocation/property accessor.
 	[{
 		line: '        let view = self.window!.contr as! UIView',
 		tokens: [
-			{ startIndex: 0, type: '' },
+			{ startIndex: 0, type: 'white.swift' },
 			{ startIndex: 8, type: 'keyword.swift' } /* 'let' */,
-			{ startIndex: 11, type: '' },
+			{ startIndex: 11, type: 'white.swift' },
 			{ startIndex: 12, type: 'identifier.swift' } /* 'view' */,
-			{ startIndex: 16, type: '' },
+			{ startIndex: 16, type: 'white.swift' },
 			{ startIndex: 17, type: 'operator.swift' } /* '=' */,
-			{ startIndex: 18, type: '' },
+			{ startIndex: 18, type: 'white.swift' },
 			{ startIndex: 19, type: 'keyword.swift' } /* 'self' */,
 			{ startIndex: 23, type: 'delimeter.swift' } /* '.' */,
 			{ startIndex: 24, type: 'type.identifier.swift' } /* 'window' */,
 			{ startIndex: 30, type: 'operator.swift' } /* '!' */,
 			{ startIndex: 31, type: 'delimeter.swift' } /* '.' */,
 			{ startIndex: 32, type: 'type.identifier.swift' } /* 'contr' */,
-			{ startIndex: 37, type: '' },
+			{ startIndex: 37, type: 'white.swift' },
 			{ startIndex: 38, type: 'keyword.swift' } /* 'as' */,
 			{ startIndex: 40, type: 'operator.swift' } /* '!' */,
-			{ startIndex: 41, type: '' },
+			{ startIndex: 41, type: 'white.swift' },
 			{ startIndex: 42, type: 'type.identifier.swift' } /* 'UIView' */
 		]
 	}]

src/swift/swift.ts

@@ -69,6 +69,7 @@ export const language = <ILanguage>{
 
 	tokenizer: {
 		root: [
+			{ include: '@whitespace' },
 			{ include: '@comment' },
 			{ include: '@attribute' },
 			{ include: '@literal' },
@@ -77,6 +78,18 @@ export const language = <ILanguage>{
 			{ include: '@symbol' },
 		],
 
+		whitespace: [
+			[/\s+/, 'white'],
+			[/"""/, 'string.quote', '@endDblDocString']
+		],
+
+		endDblDocString: [
+			[/[^"]+/, 'string'],
+			[/\\"/, 'string'],
+			[/"""/, 'string.quote', '@popall'],
+			[/"/, 'string']
+		],
+
 		symbol: [
 			[/[{}()\[\]]/, '@brackets'],
 			[/[<>](?!@symbols)/, '@brackets'],