Merge branch 'master' into redis

gulpfile.js

@@ -78,7 +78,9 @@ gulp.task('release', ['clean-release','compile'], function() {
 			bundleOne('src/yaml'),
 			bundleOne('src/solidity'),
 			bundleOne('src/sb'),
-			bundleOne('src/mysql')
+			bundleOne('src/mysql'),
+			bundleOne('src/redshift'),
+			bundleOne('src/pgsql')
 		)
 		.pipe(uglify({
 			output: {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teamsqlio/monaco-languages",
-  "version": "0.9.0",
+  "version": "0.9.3",
   "description": "Bundle of many languages for the Monaco Editor.",
   "scripts": {
     "compile": "node_modules/.bin/gulp compile",

src/monaco.contribution.ts

@@ -279,3 +279,17 @@ registerLanguage({
 	aliases: ['MySQL', 'mysql'],
 	module: './mysql'
 });
+
+registerLanguage({
+	id: 'pgsql',
+	extensions: ['.sql'],
+	aliases: ['PostgreSQL', 'postgres', 'pg', 'postgre'],
+	module: './pgsql'
+});
+
+registerLanguage({
+	id: 'redshift',
+	extensions: ['.sql'],
+	aliases: ['Redshift', 'redshift'],
+	module: './redshift'
+});

src/mysql.ts

@@ -62,9 +62,9 @@ export const language = <ILanguage>{
 		"FLOAT8", "FLUSH", "FOLLOWS", "FOR", "FORCE", "FOREIGN", "FORMAT", "FOUND", "FROM", "FULL", "FULLTEXT", "FUNCTION", "GENERAL", "GENERATED",
 		"GEOMETRY", "GEOMETRYCOLLECTION", "GET", "GET_FORMAT", "GLOBAL", "GRANT", "GRANTS", "GROUP", "GROUP_REPLICATION", "HANDLER", "HASH", "HAVING",
 		"HELP", "HIGH_PRIORITY", "HOST", "HOSTS", "HOUR", "HOUR_MICROSECOND", "HOUR_MINUTE", "HOUR_SECOND", "IDENTIFIED", "IF", "IGNORE", "IGNORE_SERVER_IDS",
-		"IMPORT", "IN", "INDEX", "INDEXES", "INFILE", "INITIAL_SIZE", "INNER", "INOUT", "INSENSITIVE", "INSERT", "INSERT_METHOD", "INSTALL", "INSTANCE",
+		"IMPORT", "INDEX", "INDEXES", "INFILE", "INITIAL_SIZE", "INNER", "INOUT", "INSENSITIVE", "INSERT", "INSERT_METHOD", "INSTALL", "INSTANCE",
 		"INT", "INT1", "INT2", "INT3", "INT4", "INT8", "INTEGER", "INTERVAL", "INTO", "INVOKER", "IO", "IO_AFTER_GTIDS", "IO_BEFORE_GTIDS", "IO_THREAD",
-		"IPC", "IS", "ISOLATION", "ISSUER", "ITERATE", "JOIN", "JSON", "KEY", "KEYS", "KEY_BLOCK_SIZE", "KILL", "LANGUAGE", "LAST", "LEADING", "LEAVE",
+		"IPC", "ISOLATION", "ISSUER", "ITERATE", "JOIN", "JSON", "KEY", "KEYS", "KEY_BLOCK_SIZE", "KILL", "LANGUAGE", "LAST", "LEADING", "LEAVE",
 		"LEAVES", "LEFT", "LESS", "LEVEL", "LIKE", "LIMIT", "LINEAR", "LINES", "LINESTRING", "LIST", "LOAD", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP", "LOCK",
 		"LOCKS", "LOGFILE", "LOGS", "LONG", "LONGBLOB", "LONGTEXT", "LOOP", "LOW_PRIORITY", "MASTER", "MASTER_AUTO_POSITION", "MASTER_BIND", "MASTER_CONNECT_RETRY",
 		"MASTER_DELAY", "MASTER_HEARTBEAT_PERIOD", "MASTER_HOST", "MASTER_LOG_FILE", "MASTER_LOG_POS", "MASTER_PASSWORD", "MASTER_PORT", "MASTER_RETRY_COUNT",
@@ -73,7 +73,7 @@ export const language = <ILanguage>{
 		"MAX_ROWS", "MAX_SIZE", "MAX_STATEMENT_TIME", "MAX_UPDATES_PER_HOUR", "MAX_USER_CONNECTIONS", "MEDIUM", "MEDIUMBLOB", "MEDIUMINT", "MEDIUMTEXT", "MEMORY",
 		"MERGE", "MESSAGE_TEXT", "MICROSECOND", "MIDDLEINT", "MIGRATE", "MINUTE", "MINUTE_MICROSECOND", "MINUTE_SECOND", "MIN_ROWS", "MOD", "MODE", "MODIFIES",
 		"MODIFY", "MONTH", "MULTILINESTRING", "MULTIPOINT", "MULTIPOLYGON", "MUTEX", "MYSQL_ERRNO", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NDB",
-		"NDBCLUSTER", "NEVER", "NEW", "NEXT", "NO", "NODEGROUP", "NONBLOCKING", "NONE", "NOT", "NO_WAIT", "NO_WRITE_TO_BINLOG", "NULL", "NUMBER", "NUMERIC",
+		"NDBCLUSTER", "NEVER", "NEW", "NEXT", "NO", "NODEGROUP", "NONBLOCKING", "NONE", "NO_WAIT", "NO_WRITE_TO_BINLOG", "NUMBER", "NUMERIC",
 		"NVARCHAR", "OFFSET", "OLD_PASSWORD", "ON", "ONE", "ONLY", "OPEN", "OPTIMIZE", "OPTIMIZER_COSTS", "OPTION", "OPTIONALLY", "OPTIONS", "OR", "ORDER",
 		"OUT", "OUTER", "OUTFILE", "OWNER", "PACK_KEYS", "PAGE", "PARSER", "PARSE_GCOL_EXPR", "PARTIAL", "PARTITION", "PARTITIONING", "PARTITIONS", "PASSWORD",
 		"PHASE", "PLUGIN", "PLUGINS", "PLUGIN_DIR", "POINT", "POLYGON", "PORT", "PRECEDES", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIVILEGES",
@@ -112,7 +112,7 @@ export const language = <ILanguage>{
 		"EXP", "EXPORT_SET", "ExteriorRing", "EXTRACT", "ExtractValue", "FIELD", "FIND_IN_SET", "FLOOR", "FORMAT", "FOUND_ROWS", "FROM_BASE64",
 		"FROM_DAYS", "FROM_UNIXTIME", "GeomCollFromText", "GeometryCollectionFromText", "GeomCollFromWKB", "GeometryCollectionFromWKB",
 		"GeometryCollection", "GeometryN", "GeometryType", "GeomFromText", "GeometryFromText", "GeomFromWKB", "GeometryFromWKB", "GET_FORMAT",
-		"GET_LOCK", "GLength", "GREATEST", "GROUP_CONCAT", "GTID_SUBSET", "GTID_SUBTRACT", "HEX", "HOUR", "IF", "IFNULL", "IN", "INET_ATON",
+		"GET_LOCK", "GLength", "GREATEST", "GROUP_CONCAT", "GTID_SUBSET", "GTID_SUBTRACT", "HEX", "HOUR", "IF", "IFNULL", "INET_ATON",
 		"INET_NTOA", "INET6_ATON", "INET6_NTOA", "INSERT", "INSTR", "InteriorRingN", "Intersects", "INTERVAL", "IS_FREE_LOCK", "IS_IPV4",
 		"IS_IPV4_COMPAT", "IS_IPV4_MAPPED", "IS_IPV6", "IS_USED_LOCK", "IsClosed", "IsEmpty", "ISNULL", "IsSimple", "JSON_APPEND", "JSON_ARRAY",
 		"JSON_ARRAY_APPEND", "JSON_ARRAY_INSERT", "JSON_CONTAINS", "JSON_CONTAINS_PATH", "JSON_DEPTH", "JSON_EXTRACT", "JSON_INSERT", "JSON_KEYS",
@@ -150,14 +150,10 @@ export const language = <ILanguage>{
 	builtinVariables: [
 		// NOT SUPPORTED
 	],
-	pseudoColumns: [
-		// NOT SUPPORTED
-	],
 	tokenizer: {
 		root: [
 			{ include: '@comments' },
 			{ include: '@whitespace' },
-			{ include: '@pseudoColumns' },
 			{ include: '@numbers' },
 			{ include: '@strings' },
 			{ include: '@complexIdentifiers' },
@@ -191,14 +187,6 @@ export const language = <ILanguage>{
 			[/\*\//, { token: 'comment.quote', next: '@pop' }],
 			[/./, 'comment']
 		],
-		pseudoColumns: [
-			[/[$][A-Za-z_][\w@#$]*/, {
-				cases: {
-					'@pseudoColumns': 'predefined',
-					'@default': 'identifier'
-				}
-			}],
-		],
 		numbers: [
 			[/0[xX][0-9a-fA-F]*/, 'number'],
 			[/[$][+-]*\d*(\.\d*)?/, 'number'],

src/pgsql.ts

@@ -211,7 +211,6 @@ export const language = <ILanguage>{
 		],
 		comments: [
 			[/--+.*/, 'comment'],
-			[/#+.*/, 'comment'],
 			[/\/\*/, { token: 'comment.quote', next: '@comment' }]
 		],
 		comment: [
@@ -237,24 +236,19 @@ export const language = <ILanguage>{
 		],
 		strings: [
 			[/'/, { token: 'string', next: '@string' }],
-			[/"/, { token: 'string', next: '@string' }]
 		],
 		string: [
 			[/[^']+/, 'string'],
-			[/[^"]+/, 'string'],
 			[/''/, 'string'],
-			[/""/, 'string'],
-			[/'/, { token: 'string', next: '@pop' }],
-			[/"/, { token: 'string', next: '@pop' }]
+			[/'/, { token: 'string', next: '@pop' }]
 		],
 		complexIdentifiers: [
-
-			[/`/, { token: 'identifier.quote', next: '@quotedIdentifier' }]
+			[/"/, { token: 'identifier.quote', next: '@quotedIdentifier' }]
 		],
 		quotedIdentifier: [
-			[/[^`]+/, 'identifier'],
-			[/``/, 'identifier'],
-			[/`/, { token: 'identifier.quote', next: '@pop' }]
+			[/[^"]+/, 'identifier'],
+			[/""/, 'identifier'],
+			[/"/, { token: 'identifier.quote', next: '@pop' }]
 		],
 		scopes: [
 			// NOT SUPPORTED

src/redshift.ts

@@ -43,7 +43,6 @@ export const language = <ILanguage>{
 		{ open: '[', close: ']', token: 'delimiter.square' },
 		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
 	],
-
 	keywords: [
 		"AES128", "AES256", "ALL", "ALLOWOVERWRITE", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "AUTHORIZATION",
 		"BACKUP", "BETWEEN", "BINARY", "BLANKSASNULL", "BOTH", "BYTEDICT", "BZIP2", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN",
@@ -169,7 +168,6 @@ export const language = <ILanguage>{
 		],
 		comments: [
 			[/--+.*/, 'comment'],
-			[/#+.*/, 'comment'],
 			[/\/\*/, { token: 'comment.quote', next: '@comment' }]
 		],
 		comment: [
@@ -195,24 +193,19 @@ export const language = <ILanguage>{
 		],
 		strings: [
 			[/'/, { token: 'string', next: '@string' }],
-			[/"/, { token: 'string', next: '@string' }]
 		],
 		string: [
 			[/[^']+/, 'string'],
-			[/[^"]+/, 'string'],
 			[/''/, 'string'],
-			[/""/, 'string'],
-			[/'/, { token: 'string', next: '@pop' }],
-			[/"/, { token: 'string', next: '@pop' }]
+			[/'/, { token: 'string', next: '@pop' }]
 		],
 		complexIdentifiers: [
-
-			[/`/, { token: 'identifier.quote', next: '@quotedIdentifier' }]
+			[/"/, { token: 'identifier.quote', next: '@quotedIdentifier' }]
 		],
 		quotedIdentifier: [
-			[/[^`]+/, 'identifier'],
-			[/``/, 'identifier'],
-			[/`/, { token: 'identifier.quote', next: '@pop' }]
+			[/[^"]+/, 'identifier'],
+			[/""/, 'identifier'],
+			[/"/, { token: 'identifier.quote', next: '@pop' }]
 		],
 		scopes: [
 			// NOT SUPPORTED

test/all.js

@@ -59,7 +59,9 @@ requirejs([
 		'out/test/yaml.test',
 		'out/test/solidity.test',
 		'out/test/sb.test',
-		'out/test/mysql.test'
+		'out/test/mysql.test',
+		'out/test/pgsql.test',
+		'out/test/redshift.test'
 	], function() {
 		run(); // We can launch the tests!
 	});

test/mysql.test.ts

@@ -513,22 +513,22 @@ testTokenization('mysql', [
 	}],
 
 	[{
-		line: 'WHERE x IS NOT NULL',
+		line: 'WHERE myfield IS NOT NULL',
 		tokens: [
 			{ startIndex: 0, type: 'keyword.sql' },
 			{ startIndex: 5, type: 'white.sql' },
 			{ startIndex: 6, type: 'identifier.sql' },
-			{ startIndex: 7, type: 'white.sql' },
-			{ startIndex: 8, type: 'operator.sql' },
-			{ startIndex: 10, type: 'white.sql' },
-			{ startIndex: 11, type: 'operator.sql' },
-			{ startIndex: 14, type: 'white.sql' },
-			{ startIndex: 15, type: 'operator.sql' }
+			{ startIndex: 13, type: 'white.sql' },
+			{ startIndex: 14, type: 'operator.sql' },
+			{ startIndex: 16, type: 'white.sql' },
+			{ startIndex: 17, type: 'operator.sql' },
+			{ startIndex: 20, type: 'white.sql' },
+			{ startIndex: 21, type: 'operator.sql' }
 		]
 	}],
 
 	[{
-		line: 'SELECT * FROM MyTable WHERE MyColumn IN (1,2)',
+		line: 'SELECT * FROM tbl WHERE MyColumn IN (1,2)',
 		tokens: [
 			{ startIndex: 0, type: 'keyword.sql' },
 			{ startIndex: 6, type: 'white.sql' },
@@ -537,21 +537,18 @@ testTokenization('mysql', [
 			{ startIndex: 9, type: 'keyword.sql' },
 			{ startIndex: 13, type: 'white.sql' },
 			{ startIndex: 14, type: 'identifier.sql' },
-			{ startIndex: 17, type: 'delimiter.sql' },
-			{ startIndex: 18, type: 'identifier.sql' },
-			{ startIndex: 25, type: 'white.sql' },
-			{ startIndex: 26, type: 'keyword.sql' },
-			{ startIndex: 31, type: 'white.sql' },
-			{ startIndex: 32, type: 'identifier.sql' },
-			{ startIndex: 40, type: 'white.sql' },
-			{ startIndex: 41, type: 'operator.sql' },
-			{ startIndex: 43, type: 'white.sql' },
-			{ startIndex: 44, type: 'delimiter.parenthesis.sql' },
-			{ startIndex: 45, type: 'number.sql' },
-			{ startIndex: 46, type: 'delimiter.sql' },
-			{ startIndex: 47, type: 'number.sql' },
-			{ startIndex: 48, type: 'delimiter.parenthesis.sql' }
+			{ startIndex: 17, type: 'white.sql' },
+			{ startIndex: 18, type: 'keyword.sql' },
+			{ startIndex: 23, type: 'white.sql' },
+			{ startIndex: 24, type: 'identifier.sql' },
+			{ startIndex: 32, type: 'white.sql' },
+			{ startIndex: 33, type: 'operator.sql' },
+			{ startIndex: 35, type: 'white.sql' },
+			{ startIndex: 36, type: 'delimiter.parenthesis.sql' },
+			{ startIndex: 37, type: 'number.sql' },
+			{ startIndex: 38, type: 'delimiter.sql' },
+			{ startIndex: 39, type: 'number.sql' },
+			{ startIndex: 40, type: 'delimiter.parenthesis.sql' }
 		]
 	}]
-
 ]);

test/pgsql.test.ts

@@ -0,0 +1,555 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from './testRunner';
+
+testTokenization('sql', [
+	// Comments
+	[{
+		line: '-- a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.sql' }
+		]
+	}],
+
+	[{
+		line: '---sticky -- comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.sql' }
+		]
+	}],
+
+	[{
+		line: '-almost a comment',
+		tokens: [
+			{ startIndex: 0, type: 'operator.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.sql' },
+			{ startIndex: 9, type: 'white.sql' },
+			{ startIndex: 10, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '/* a full line comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.quote.sql' },
+			{ startIndex: 2, type: 'comment.sql' },
+			{ startIndex: 23, type: 'comment.quote.sql' }
+		]
+	}],
+
+	[{
+		line: '/* /// *** /// */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.quote.sql' },
+			{ startIndex: 2, type: 'comment.sql' },
+			{ startIndex: 15, type: 'comment.quote.sql' }
+		]
+	}],
+
+	[{
+		line: 'declare _x int = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.sql' },
+			{ startIndex: 10, type: 'white.sql' },
+			{ startIndex: 11, type: 'keyword.sql' },
+			{ startIndex: 14, type: 'white.sql' },
+			{ startIndex: 15, type: 'operator.sql' },
+			{ startIndex: 16, type: 'white.sql' },
+			{ startIndex: 17, type: 'comment.quote.sql' },
+			{ startIndex: 19, type: 'comment.sql' },
+			{ startIndex: 37, type: 'comment.quote.sql' },
+			{ startIndex: 39, type: 'white.sql' },
+			{ startIndex: 40, type: 'number.sql' },
+			{ startIndex: 41, type: 'delimiter.sql' }
+		]
+	}],
+
+	// Not supporting nested comments, as nested comments seem to not be standard?
+	// i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic
+	[{
+		line: '_x=/* a /* nested comment  1*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 3, type: 'comment.quote.sql' },
+			{ startIndex: 5, type: 'comment.sql' },
+			{ startIndex: 28, type: 'comment.quote.sql' },
+			{ startIndex: 30, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '_x=/* another comment */ 1*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 3, type: 'comment.quote.sql' },
+			{ startIndex: 5, type: 'comment.sql' },
+			{ startIndex: 22, type: 'comment.quote.sql' },
+			{ startIndex: 24, type: 'white.sql' },
+			{ startIndex: 25, type: 'number.sql' },
+			{ startIndex: 26, type: 'operator.sql' },
+			{ startIndex: 28, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '_x=/*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 3, type: 'comment.quote.sql' },
+			{ startIndex: 5, type: 'comment.sql' }
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '-123',
+		tokens: [
+			{ startIndex: 0, type: 'operator.sql' },
+			{ startIndex: 1, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0xaBc123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0XaBc123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0x0',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0xAB_CD',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' },
+			{ startIndex: 4, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '$',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$-123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$-+-123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$123.5678',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$0.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$99.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$0.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$.0',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '123.5678',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '99.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.0',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1E-2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1E+2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0.1E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1.E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.1E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	// Identifiers
+	[{
+		line: '_abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '#abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '##abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '@abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '@@abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '$abc',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '$nonexistent',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '@@nonexistent',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'declare "abc 321";',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.quote.sql' },
+			{ startIndex: 9, type: 'identifier.sql' },
+			{ startIndex: 16, type: 'identifier.quote.sql' },
+			{ startIndex: 17, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc"" 321 "" xyz"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 17, type: 'identifier.quote.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'declare "abc 321";',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.quote.sql' },
+			{ startIndex: 9, type: 'identifier.sql' },
+			{ startIndex: 16, type: 'identifier.quote.sql' },
+			{ startIndex: 17, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc"" 321 "" xyz"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 17, type: 'identifier.quote.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'int',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' }
+		]
+	}],
+
+	[{
+		line: '"int"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 4, type: 'identifier.quote.sql' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: 'declare _x=\'a string\';',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.sql' },
+			{ startIndex: 10, type: 'operator.sql' },
+			{ startIndex: 11, type: 'string.sql' },
+			{ startIndex: 21, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '\'a \'\' string with quotes\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.sql' },
+		]
+	}],
+
+	[{
+		line: '\'a -- string with comment\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.sql' },
+		]
+	}],
+
+	[{
+		line: '\'a endless string',
+		tokens: [
+			{ startIndex: 0, type: 'string.sql' },
+		]
+	}],
+
+	// Operators
+	[{
+		line: 'x=x+1',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 1, type: 'operator.sql' },
+			{ startIndex: 2, type: 'identifier.sql' },
+			{ startIndex: 3, type: 'operator.sql' },
+			{ startIndex: 4, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '_x^=_x',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 4, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'WHERE x IS NOT NULL',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 5, type: 'white.sql' },
+			{ startIndex: 6, type: 'identifier.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'operator.sql' },
+			{ startIndex: 10, type: 'white.sql' },
+			{ startIndex: 11, type: 'operator.sql' },
+			{ startIndex: 14, type: 'white.sql' },
+			{ startIndex: 15, type: 'operator.sql' }
+		]
+	}],
+
+	[{
+		line: 'SELECT * FROM sch.MyTable WHERE MyColumn IN (1,2)',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 6, type: 'white.sql' },
+			{ startIndex: 7, type: 'operator.sql' },
+			{ startIndex: 8, type: 'white.sql' },
+			{ startIndex: 9, type: 'keyword.sql' },
+			{ startIndex: 13, type: 'white.sql' },
+			{ startIndex: 14, type: 'identifier.sql' },
+			{ startIndex: 17, type: 'delimiter.sql' },
+			{ startIndex: 18, type: 'identifier.sql' },
+			{ startIndex: 25, type: 'white.sql' },
+			{ startIndex: 26, type: 'keyword.sql' },
+			{ startIndex: 31, type: 'white.sql' },
+			{ startIndex: 32, type: 'identifier.sql' },
+			{ startIndex: 40, type: 'white.sql' },
+			{ startIndex: 41, type: 'operator.sql' },
+			{ startIndex: 43, type: 'white.sql' },
+			{ startIndex: 44, type: 'delimiter.parenthesis.sql' },
+			{ startIndex: 45, type: 'number.sql' },
+			{ startIndex: 46, type: 'delimiter.sql' },
+			{ startIndex: 47, type: 'number.sql' },
+			{ startIndex: 48, type: 'delimiter.parenthesis.sql' }
+		]
+	}]
+]);

test/redshift.test.ts

@@ -0,0 +1,555 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from './testRunner';
+
+testTokenization('sql', [
+	// Comments
+	[{
+		line: '-- a comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.sql' }
+		]
+	}],
+
+	[{
+		line: '---sticky -- comment',
+		tokens: [
+			{ startIndex: 0, type: 'comment.sql' }
+		]
+	}],
+
+	[{
+		line: '-almost a comment',
+		tokens: [
+			{ startIndex: 0, type: 'operator.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.sql' },
+			{ startIndex: 9, type: 'white.sql' },
+			{ startIndex: 10, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '/* a full line comment */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.quote.sql' },
+			{ startIndex: 2, type: 'comment.sql' },
+			{ startIndex: 23, type: 'comment.quote.sql' }
+		]
+	}],
+
+	[{
+		line: '/* /// *** /// */',
+		tokens: [
+			{ startIndex: 0, type: 'comment.quote.sql' },
+			{ startIndex: 2, type: 'comment.sql' },
+			{ startIndex: 15, type: 'comment.quote.sql' }
+		]
+	}],
+
+	[{
+		line: 'declare _x int = /* a simple comment */ 1;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.sql' },
+			{ startIndex: 10, type: 'white.sql' },
+			{ startIndex: 11, type: 'keyword.sql' },
+			{ startIndex: 14, type: 'white.sql' },
+			{ startIndex: 15, type: 'operator.sql' },
+			{ startIndex: 16, type: 'white.sql' },
+			{ startIndex: 17, type: 'comment.quote.sql' },
+			{ startIndex: 19, type: 'comment.sql' },
+			{ startIndex: 37, type: 'comment.quote.sql' },
+			{ startIndex: 39, type: 'white.sql' },
+			{ startIndex: 40, type: 'number.sql' },
+			{ startIndex: 41, type: 'delimiter.sql' }
+		]
+	}],
+
+	// Not supporting nested comments, as nested comments seem to not be standard?
+	// i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic
+	[{
+		line: '_x=/* a /* nested comment  1*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 3, type: 'comment.quote.sql' },
+			{ startIndex: 5, type: 'comment.sql' },
+			{ startIndex: 28, type: 'comment.quote.sql' },
+			{ startIndex: 30, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '_x=/* another comment */ 1*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 3, type: 'comment.quote.sql' },
+			{ startIndex: 5, type: 'comment.sql' },
+			{ startIndex: 22, type: 'comment.quote.sql' },
+			{ startIndex: 24, type: 'white.sql' },
+			{ startIndex: 25, type: 'number.sql' },
+			{ startIndex: 26, type: 'operator.sql' },
+			{ startIndex: 28, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '_x=/*/;',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 3, type: 'comment.quote.sql' },
+			{ startIndex: 5, type: 'comment.sql' }
+		]
+	}],
+
+	// Numbers
+	[{
+		line: '123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '-123',
+		tokens: [
+			{ startIndex: 0, type: 'operator.sql' },
+			{ startIndex: 1, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0xaBc123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0XaBc123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0x',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0x0',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0xAB_CD',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' },
+			{ startIndex: 4, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '$',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$-123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$-+-123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$123.5678',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$0.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$99.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$0.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '$.0',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '123',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '123.5678',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.99',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '99.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0.',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.0',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1E-2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1E+2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '0.1E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '1.E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '.1E2',
+		tokens: [
+			{ startIndex: 0, type: 'number.sql' }
+		]
+	}],
+
+	// Identifiers
+	[{
+		line: '_abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '#abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '##abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '@abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '@@abc$01',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '$abc',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '$nonexistent',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: '@@nonexistent',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'declare "abc 321";',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.quote.sql' },
+			{ startIndex: 9, type: 'identifier.sql' },
+			{ startIndex: 16, type: 'identifier.quote.sql' },
+			{ startIndex: 17, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc"" 321 "" xyz"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 17, type: 'identifier.quote.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'declare "abc 321";',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.quote.sql' },
+			{ startIndex: 9, type: 'identifier.sql' },
+			{ startIndex: 16, type: 'identifier.quote.sql' },
+			{ startIndex: 17, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc"" 321 "" xyz"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 17, type: 'identifier.quote.sql' }
+		]
+	}],
+
+	[{
+		line: '"abc',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'int',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' }
+		]
+	}],
+
+	[{
+		line: '"int"',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.quote.sql' },
+			{ startIndex: 1, type: 'identifier.sql' },
+			{ startIndex: 4, type: 'identifier.quote.sql' }
+		]
+	}],
+
+	// Strings
+	[{
+		line: 'declare _x=\'a string\';',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'identifier.sql' },
+			{ startIndex: 10, type: 'operator.sql' },
+			{ startIndex: 11, type: 'string.sql' },
+			{ startIndex: 21, type: 'delimiter.sql' }
+		]
+	}],
+
+	[{
+		line: '\'a \'\' string with quotes\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.sql' },
+		]
+	}],
+
+	[{
+		line: '\'a -- string with comment\'',
+		tokens: [
+			{ startIndex: 0, type: 'string.sql' },
+		]
+	}],
+
+	[{
+		line: '\'a endless string',
+		tokens: [
+			{ startIndex: 0, type: 'string.sql' },
+		]
+	}],
+
+	// Operators
+	[{
+		line: 'x=x+1',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 1, type: 'operator.sql' },
+			{ startIndex: 2, type: 'identifier.sql' },
+			{ startIndex: 3, type: 'operator.sql' },
+			{ startIndex: 4, type: 'number.sql' }
+		]
+	}],
+
+	[{
+		line: '_x^=_x',
+		tokens: [
+			{ startIndex: 0, type: 'identifier.sql' },
+			{ startIndex: 2, type: 'operator.sql' },
+			{ startIndex: 4, type: 'identifier.sql' }
+		]
+	}],
+
+	[{
+		line: 'WHERE x IS NOT NULL',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 5, type: 'white.sql' },
+			{ startIndex: 6, type: 'identifier.sql' },
+			{ startIndex: 7, type: 'white.sql' },
+			{ startIndex: 8, type: 'operator.sql' },
+			{ startIndex: 10, type: 'white.sql' },
+			{ startIndex: 11, type: 'operator.sql' },
+			{ startIndex: 14, type: 'white.sql' },
+			{ startIndex: 15, type: 'operator.sql' }
+		]
+	}],
+
+	[{
+		line: 'SELECT * FROM sch.MyTable WHERE MyColumn IN (1,2)',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.sql' },
+			{ startIndex: 6, type: 'white.sql' },
+			{ startIndex: 7, type: 'operator.sql' },
+			{ startIndex: 8, type: 'white.sql' },
+			{ startIndex: 9, type: 'keyword.sql' },
+			{ startIndex: 13, type: 'white.sql' },
+			{ startIndex: 14, type: 'identifier.sql' },
+			{ startIndex: 17, type: 'delimiter.sql' },
+			{ startIndex: 18, type: 'identifier.sql' },
+			{ startIndex: 25, type: 'white.sql' },
+			{ startIndex: 26, type: 'keyword.sql' },
+			{ startIndex: 31, type: 'white.sql' },
+			{ startIndex: 32, type: 'identifier.sql' },
+			{ startIndex: 40, type: 'white.sql' },
+			{ startIndex: 41, type: 'operator.sql' },
+			{ startIndex: 43, type: 'white.sql' },
+			{ startIndex: 44, type: 'delimiter.parenthesis.sql' },
+			{ startIndex: 45, type: 'number.sql' },
+			{ startIndex: 46, type: 'delimiter.sql' },
+			{ startIndex: 47, type: 'number.sql' },
+			{ startIndex: 48, type: 'delimiter.parenthesis.sql' }
+		]
+	}]
+]);