Lotsa upates getting tests to pass

src/kotlin/kotlin.test.ts

@@ -0,0 +1,647 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('kotlin', [
+	// Comments - single line
+	[{
+		line: '//',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: '    // a comment',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 4, type: 'comment.kt' }
+		]
+	}],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[{
+		line: '/* //*/ a',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '// a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: '//sticky comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: '/almost a comment',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.kt' },
+			{ startIndex: 1, type: 'identifier.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.kt' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '1 / 2; /* comment',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.kt' },
+			{ startIndex: 5, type: 'delimiter.kt' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'comment.kt' }
+		]
+	}],
+
+	// [{
+	// 	line: 'var x = 1 // my comment // is a nice one',
+	// 	tokens: [
+	// 		{ startIndex: 0, type: 'keyword.var.kt' },
+	// 		{ startIndex: 3, type: '' },
+	// 		{ startIndex: 4, type: 'identifier.kt' },
+	// 		{ startIndex: 5, type: '' },
+	// 		{ startIndex: 6, type: 'delimiter.kt' },
+	// 		{ startIndex: 7, type: '' },
+	// 		{ startIndex: 8, type: 'number.kt' },
+	// 		{ startIndex: 9, type: '' },
+	// 		{ startIndex: 10, type: 'comment.kt' },
+	// 		{ startIndex: 12, type: '' },
+	// 		{ startIndex: 13, type: 'comment.kt' }
+	// 	]
+	// }],
+
+	// Comments - range comment, single line
+	[{
+		line: '/* a simple comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.kt' }
+		]
+	}],
+
+	[{
+		line: 'var x = /* a simple comment */ 1',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.kt' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'number.kt' },
+		]
+	}],
+
+	[{
+		line: 'var x = /* comment */ 1; */',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.kt' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'number.kt' },
+			{ startIndex: 23, type: 'delimiter.kt' },
+			{ startIndex: 24, type: '' }
+		]
+	}],
+
+	[{
+		line: 'x = /**/',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.kt' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'comment.kt' },
+		]
+	}],
+
+	[{
+		line: 'var x = /** start a Java Doc comment',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.var.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.kt' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'delimiter.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: ' a ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: 'and end it */ 2',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'number.kt' },
+		]
+	}],
+
+	[{
+		line: '/** start of Java Doc',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: 'a comment between without a star',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}, {
+		line: 'end of multiline comment*/',
+		tokens: [
+			{ startIndex: 0, type: 'comment.doc.kt' }
+		]
+	}],
+
+	// Keywords
+	[{
+		line: 'package test; class Program { fun main(args: String[]) {} } }',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.package.kt' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'identifier.kt' },
+			{ startIndex: 12, type: 'delimiter.kt' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 14, type: 'keyword.class.kt' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'identifier.kt' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 28, type: 'delimiter.curly.kt' },
+			{ startIndex: 29, type: '' },
+			{ startIndex: 30, type: 'keyword.fun.kt' },
+			{ startIndex: 36, type: '' },
+			{ startIndex: 42, type: 'identifier.kt' },
+			{ startIndex: 46, type: 'delimiter.parenthesis.kt' },
+			{ startIndex: 47, type: 'identifier.kt' },
+			{ startIndex: 53, type: 'delimiter.square.kt' },
+			{ startIndex: 55, type: '' },
+			{ startIndex: 56, type: 'identifier.kt' },
+			{ startIndex: 60, type: 'delimiter.parenthesis.kt' },
+			{ startIndex: 61, type: '' },
+			{ startIndex: 62, type: 'delimiter.curly.kt' },
+			{ startIndex: 64, type: '' },
+			{ startIndex: 65, type: 'delimiter.curly.kt' },
+			{ startIndex: 66, type: '' },
+			{ startIndex: 67, type: 'delimiter.curly.kt' }
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '0',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '0.10',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0x123',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.kt' }
+		]
+	}],
+
+	[{
+		line: '0x5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.kt' }
+		]
+	}],
+
+	[{
+		line: '023L',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' }
+		]
+	}],
+
+	[{
+		line: '0123l',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' }
+		]
+	}],
+
+	[{
+		line: '05_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' }
+		]
+	}],
+
+	[{
+		line: '0b1010_0101',
+		tokens: [
+			{ startIndex: 0, type: 'number.binary.kt' }
+		]
+	}],
+
+	[{
+		line: '0B001',
+		tokens: [
+			{ startIndex: 0, type: 'number.binary.kt' }
+		]
+	}],
+
+	[{
+		line: '10e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '10f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5e3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5e-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5E3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5E-3',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5F',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5f',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72E3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72E3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72E-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72e3D',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72e3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '1.72e-3d',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '23L',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '23l',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '0_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '5_2',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '5_______2',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '3_.1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: 'number.float.kt' }
+		]
+	}],
+
+	[{
+		line: '3._1415F',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'delimiter.kt' },
+			{ startIndex: 2, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '999_99_9999_L',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 11, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 2, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0_x52',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0x_52',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0x52_',
+		tokens: [
+			{ startIndex: 0, type: 'number.hex.kt' },
+			{ startIndex: 4, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '052_',
+		tokens: [
+			{ startIndex: 0, type: 'number.octal.kt' },
+			{ startIndex: 3, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '23.5L',
+		tokens: [
+			{ startIndex: 0, type: 'number.float.kt' },
+			{ startIndex: 4, type: 'identifier.kt' }
+		]
+	}],
+
+	[{
+		line: '0+0',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: 'delimiter.kt' },
+			{ startIndex: 2, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '100+10',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 3, type: 'delimiter.kt' },
+			{ startIndex: 4, type: 'number.kt' }
+		]
+	}],
+
+	[{
+		line: '0 + 0',
+		tokens: [
+			{ startIndex: 0, type: 'number.kt' },
+			{ startIndex: 1, type: '' },
+			{ startIndex: 2, type: 'delimiter.kt' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'number.kt' }
+		]
+	}],
+
+	// single line Strings
+	[{
+		line: 'var s = "I\'m a Kotlin String";',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.kt' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'identifier.kt' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'delimiter.kt' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'string.kt' },
+			{ startIndex: 30, type: 'delimiter.kt' }
+		]
+	}],
+
+	[{
+		line: 'String s = "concatenated" + " String" ;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.kt' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'identifier.kt' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'delimiter.kt' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'string.kt' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'delimiter.kt' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 28, type: 'string.kt' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'delimiter.kt' }
+		]
+	}],
+
+	[{
+		line: '"quote in a string"',
+		tokens: [
+			{ startIndex: 0, type: 'string.kt' }
+		]
+	}],
+
+	[{
+		line: '"escaping \\"quotes\\" is cool"',
+		tokens: [
+			{ startIndex: 0, type: 'string.kt' },
+			{ startIndex: 10, type: 'string.escape.kt' },
+			{ startIndex: 12, type: 'string.kt' },
+			{ startIndex: 18, type: 'string.escape.kt' },
+			{ startIndex: 20, type: 'string.kt' }
+		]
+	}],
+
+	[{
+		line: '"\\"',
+		tokens: [
+			{ startIndex: 0, type: 'string.invalid.kt' }
+		]
+	}],
+
+	// Annotations
+	[{
+		line: '@',
+		tokens: [
+			{ startIndex: 0, type: '' }
+		]
+	}],
+
+	[{
+		line: '@Override',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.kt' }
+		]
+	}],
+
+	[{
+		line: '@SuppressWarnings(value = "aString")',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.kt' },
+			{ startIndex: 17, type: 'delimiter.parenthesis.kt' },
+			{ startIndex: 18, type: 'identifier.kt' },
+			{ startIndex: 23, type: '' },
+			{ startIndex: 24, type: 'delimiter.kt' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'string.kt' },
+			{ startIndex: 35, type: 'delimiter.parenthesis.kt' }
+		]
+	}],
+
+	[{
+		line: '@ AnnotationWithKeywordAfter private',
+		tokens: [
+			{ startIndex: 0, type: 'annotation.kt' },
+			{ startIndex: 28, type: '' },
+			{ startIndex: 29, type: 'keyword.private.kt' }
+		]
+	}]
+]);
+

src/kotlin/kotlin.ts

@@ -13,7 +13,7 @@ export const conf: IRichLanguageConfiguration = {
 	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
 	comments: {
 		lineComment: '//',
-		blockComment: ['/**', '*/'],
+		blockComment: ['/*', '*/'],
 	},
 	brackets: [
 		['{', '}'],
@@ -45,7 +45,7 @@ export const conf: IRichLanguageConfiguration = {
 
 export const language = <ILanguage>{
 	defaultToken: '',
-	tokenPostfix: '.java',
+	tokenPostfix: '.kt',
 
 	keywords: [
 		'as', 'as?', 'break', 'class', 'continue', 'do', 'else', 'false', 'for', 'fun', 'if',

test/setup.js

@@ -42,6 +42,7 @@ define(['require'], function () {
 			'release/dev/html/html.test',
 			'release/dev/java/java.test',
 			'release/dev/javascript/javascript.test',
+			'release/dev/kotlin/kotlin.test',
 			'release/dev/less/less.test',
 			'release/dev/lua/lua.test',
 			'release/dev/markdown/markdown.test',