[clojure] Improve Clojure syntax highlighting.

This commit:
- Updates the keyword list with public symbols from `clojure.core` for
Clojure 1.9.
- Improves syntax highlighting of symbols, strings, character literals,
and number literals.
- Adds rules for syntax highlighting Clojure keywords and reader macro
characters.
- Fixes an issue where forms following a single semicolon (`;`) are not
marked as comments.
- Fixes an issue where parts of multi-line strings were not correctly
syntax highlighted.
- Adds additional test coverage for tokenization code.

src/clojure/clojure.test.ts

@@ -5,86 +5,775 @@
 
 'use strict';
 
-import { testTokenization } from '../test/testRunner';
+import {ITestItem, testTokenization} from '../test/testRunner';
 
-testTokenization('clojure', [
-	// Keywords
-	[
-		{
-			line: 'defmacro some',
-			tokens: [
-				{ startIndex: 0, type: 'keyword.clj' },
-				{ startIndex: 8, type: 'white.clj' },
-				{ startIndex: 9, type: 'variable.clj' },
-			],
-		},
+const specialForms = [
+	'.',
+	'catch',
+	'def',
+	'do',
+	'if',
+	'monitor-enter',
+	'monitor-exit',
+	'new',
+	'quote',
+	'recur',
+	'set!',
+	'throw',
+	'try',
+	'var',
+];
 
-		{
-			line: 'comment "text comment"',
-			tokens: [
-				{ startIndex: 0, type: 'keyword.clj' },
-				{ startIndex: 7, type: 'white.clj' },
-				{ startIndex: 8, type: 'string.clj' },
-			],
-		},
-		{
-			line: 'in-ns "user',
-			tokens: [
-				{ startIndex: 0, type: 'keyword.clj' },
-				{ startIndex: 5, type: 'white.clj' },
-				{ startIndex: 6, type: 'string.clj' },
-			],
-		},
-	],
+const coreSymbols = [
+	'*',
+	'*\'',
+	'*1',
+	'*2',
+	'*3',
+	'*agent*',
+	'*allow-unresolved-vars*',
+	'*assert*',
+	'*clojure-version*',
+	'*command-line-args*',
+	'*compile-files*',
+	'*compile-path*',
+	'*compiler-options*',
+	'*data-readers*',
+	'*default-data-reader-fn*',
+	'*e',
+	'*err*',
+	'*file*',
+	'*flush-on-newline*',
+	'*fn-loader*',
+	'*in*',
+	'*math-context*',
+	'*ns*',
+	'*out*',
+	'*print-dup*',
+	'*print-length*',
+	'*print-level*',
+	'*print-meta*',
+	'*print-namespace-maps*',
+	'*print-readably*',
+	'*read-eval*',
+	'*reader-resolver*',
+	'*source-path*',
+	'*suppress-read*',
+	'*unchecked-math*',
+	'*use-context-classloader*',
+	'*verbose-defrecords*',
+	'*warn-on-reflection*',
+	'+',
+	'+\'',
+	'-',
+	'-\'',
+	'->',
+	'->>',
+	'->ArrayChunk',
+	'->Eduction',
+	'->Vec',
+	'->VecNode',
+	'->VecSeq',
+	'-cache-protocol-fn',
+	'-reset-methods',
+	'..',
+	'/',
+	'<',
+	'<=',
+	'=',
+	'==',
+	'>',
+	'>=',
+	'EMPTY-NODE',
+	'Inst',
+	'StackTraceElement->vec',
+	'Throwable->map',
+	'accessor',
+	'aclone',
+	'add-classpath',
+	'add-watch',
+	'agent',
+	'agent-error',
+	'agent-errors',
+	'aget',
+	'alength',
+	'alias',
+	'all-ns',
+	'alter',
+	'alter-meta!',
+	'alter-var-root',
+	'amap',
+	'ancestors',
+	'and',
+	'any?',
+	'apply',
+	'areduce',
+	'array-map',
+	'as->',
+	'aset',
+	'aset-boolean',
+	'aset-byte',
+	'aset-char',
+	'aset-double',
+	'aset-float',
+	'aset-int',
+	'aset-long',
+	'aset-short',
+	'assert',
+	'assoc',
+	'assoc!',
+	'assoc-in',
+	'associative?',
+	'atom',
+	'await',
+	'await-for',
+	'await1',
+	'bases',
+	'bean',
+	'bigdec',
+	'bigint',
+	'biginteger',
+	'binding',
+	'bit-and',
+	'bit-and-not',
+	'bit-clear',
+	'bit-flip',
+	'bit-not',
+	'bit-or',
+	'bit-set',
+	'bit-shift-left',
+	'bit-shift-right',
+	'bit-test',
+	'bit-xor',
+	'boolean',
+	'boolean-array',
+	'boolean?',
+	'booleans',
+	'bound-fn',
+	'bound-fn*',
+	'bound?',
+	'bounded-count',
+	'butlast',
+	'byte',
+	'byte-array',
+	'bytes',
+	'bytes?',
+	'case',
+	'cast',
+	'cat',
+	'char',
+	'char-array',
+	'char-escape-string',
+	'char-name-string',
+	'char?',
+	'chars',
+	'chunk',
+	'chunk-append',
+	'chunk-buffer',
+	'chunk-cons',
+	'chunk-first',
+	'chunk-next',
+	'chunk-rest',
+	'chunked-seq?',
+	'class',
+	'class?',
+	'clear-agent-errors',
+	'clojure-version',
+	'coll?',
+	'comment',
+	'commute',
+	'comp',
+	'comparator',
+	'compare',
+	'compare-and-set!',
+	'compile',
+	'complement',
+	'completing',
+	'concat',
+	'cond',
+	'cond->',
+	'cond->>',
+	'condp',
+	'conj',
+	'conj!',
+	'cons',
+	'constantly',
+	'construct-proxy',
+	'contains?',
+	'count',
+	'counted?',
+	'create-ns',
+	'create-struct',
+	'cycle',
+	'dec',
+	'dec\'',
+	'decimal?',
+	'declare',
+	'dedupe',
+	'default-data-readers',
+	'definline',
+	'definterface',
+	'defmacro',
+	'defmethod',
+	'defmulti',
+	'defn',
+	'defn-',
+	'defonce',
+	'defprotocol',
+	'defrecord',
+	'defstruct',
+	'deftype',
+	'delay',
+	'delay?',
+	'deliver',
+	'denominator',
+	'deref',
+	'derive',
+	'descendants',
+	'destructure',
+	'disj',
+	'disj!',
+	'dissoc',
+	'dissoc!',
+	'distinct',
+	'distinct?',
+	'doall',
+	'dorun',
+	'doseq',
+	'dosync',
+	'dotimes',
+	'doto',
+	'double',
+	'double-array',
+	'double?',
+	'doubles',
+	'drop',
+	'drop-last',
+	'drop-while',
+	'eduction',
+	'empty',
+	'empty?',
+	'ensure',
+	'ensure-reduced',
+	'enumeration-seq',
+	'error-handler',
+	'error-mode',
+	'eval',
+	'even?',
+	'every-pred',
+	'every?',
+	'ex-data',
+	'ex-info',
+	'extend',
+	'extend-protocol',
+	'extend-type',
+	'extenders',
+	'extends?',
+	'false?',
+	'ffirst',
+	'file-seq',
+	'filter',
+	'filterv',
+	'find',
+	'find-keyword',
+	'find-ns',
+	'find-protocol-impl',
+	'find-protocol-method',
+	'find-var',
+	'first',
+	'flatten',
+	'float',
+	'float-array',
+	'float?',
+	'floats',
+	'flush',
+	'fn',
+	'fn?',
+	'fnext',
+	'fnil',
+	'for',
+	'force',
+	'format',
+	'frequencies',
+	'future',
+	'future-call',
+	'future-cancel',
+	'future-cancelled?',
+	'future-done?',
+	'future?',
+	'gen-class',
+	'gen-interface',
+	'gensym',
+	'get',
+	'get-in',
+	'get-method',
+	'get-proxy-class',
+	'get-thread-bindings',
+	'get-validator',
+	'group-by',
+	'halt-when',
+	'hash',
+	'hash-combine',
+	'hash-map',
+	'hash-ordered-coll',
+	'hash-set',
+	'hash-unordered-coll',
+	'ident?',
+	'identical?',
+	'identity',
+	'if-let',
+	'if-not',
+	'if-some',
+	'ifn?',
+	'import',
+	'in-ns',
+	'inc',
+	'inc\'',
+	'indexed?',
+	'init-proxy',
+	'inst-ms',
+	'inst-ms*',
+	'inst?',
+	'instance?',
+	'int',
+	'int-array',
+	'int?',
+	'integer?',
+	'interleave',
+	'intern',
+	'interpose',
+	'into',
+	'into-array',
+	'ints',
+	'io!',
+	'isa?',
+	'iterate',
+	'iterator-seq',
+	'juxt',
+	'keep',
+	'keep-indexed',
+	'key',
+	'keys',
+	'keyword',
+	'keyword?',
+	'last',
+	'lazy-cat',
+	'lazy-seq',
+	'let',
+	'letfn',
+	'line-seq',
+	'list',
+	'list*',
+	'list?',
+	'load',
+	'load-file',
+	'load-reader',
+	'load-string',
+	'loaded-libs',
+	'locking',
+	'long',
+	'long-array',
+	'longs',
+	'loop',
+	'macroexpand',
+	'macroexpand-1',
+	'make-array',
+	'make-hierarchy',
+	'map',
+	'map-entry?',
+	'map-indexed',
+	'map?',
+	'mapcat',
+	'mapv',
+	'max',
+	'max-key',
+	'memfn',
+	'memoize',
+	'merge',
+	'merge-with',
+	'meta',
+	'method-sig',
+	'methods',
+	'min',
+	'min-key',
+	'mix-collection-hash',
+	'mod',
+	'munge',
+	'name',
+	'namespace',
+	'namespace-munge',
+	'nat-int?',
+	'neg-int?',
+	'neg?',
+	'newline',
+	'next',
+	'nfirst',
+	'nil?',
+	'nnext',
+	'not',
+	'not-any?',
+	'not-empty',
+	'not-every?',
+	'not=',
+	'ns',
+	'ns-aliases',
+	'ns-imports',
+	'ns-interns',
+	'ns-map',
+	'ns-name',
+	'ns-publics',
+	'ns-refers',
+	'ns-resolve',
+	'ns-unalias',
+	'ns-unmap',
+	'nth',
+	'nthnext',
+	'nthrest',
+	'num',
+	'number?',
+	'numerator',
+	'object-array',
+	'odd?',
+	'or',
+	'parents',
+	'partial',
+	'partition',
+	'partition-all',
+	'partition-by',
+	'pcalls',
+	'peek',
+	'persistent!',
+	'pmap',
+	'pop',
+	'pop!',
+	'pop-thread-bindings',
+	'pos-int?',
+	'pos?',
+	'pr',
+	'pr-str',
+	'prefer-method',
+	'prefers',
+	'primitives-classnames',
+	'print',
+	'print-ctor',
+	'print-dup',
+	'print-method',
+	'print-simple',
+	'print-str',
+	'printf',
+	'println',
+	'println-str',
+	'prn',
+	'prn-str',
+	'promise',
+	'proxy',
+	'proxy-call-with-super',
+	'proxy-mappings',
+	'proxy-name',
+	'proxy-super',
+	'push-thread-bindings',
+	'pvalues',
+	'qualified-ident?',
+	'qualified-keyword?',
+	'qualified-symbol?',
+	'quot',
+	'rand',
+	'rand-int',
+	'rand-nth',
+	'random-sample',
+	'range',
+	'ratio?',
+	'rational?',
+	'rationalize',
+	're-find',
+	're-groups',
+	're-matcher',
+	're-matches',
+	're-pattern',
+	're-seq',
+	'read',
+	'read-line',
+	'read-string',
+	'reader-conditional',
+	'reader-conditional?',
+	'realized?',
+	'record?',
+	'reduce',
+	'reduce-kv',
+	'reduced',
+	'reduced?',
+	'reductions',
+	'ref',
+	'ref-history-count',
+	'ref-max-history',
+	'ref-min-history',
+	'ref-set',
+	'refer',
+	'refer-clojure',
+	'reify',
+	'release-pending-sends',
+	'rem',
+	'remove',
+	'remove-all-methods',
+	'remove-method',
+	'remove-ns',
+	'remove-watch',
+	'repeat',
+	'repeatedly',
+	'replace',
+	'replicate',
+	'require',
+	'reset!',
+	'reset-meta!',
+	'reset-vals!',
+	'resolve',
+	'rest',
+	'restart-agent',
+	'resultset-seq',
+	'reverse',
+	'reversible?',
+	'rseq',
+	'rsubseq',
+	'run!',
+	'satisfies?',
+	'second',
+	'select-keys',
+	'send',
+	'send-off',
+	'send-via',
+	'seq',
+	'seq?',
+	'seqable?',
+	'seque',
+	'sequence',
+	'sequential?',
+	'set',
+	'set-agent-send-executor!',
+	'set-agent-send-off-executor!',
+	'set-error-handler!',
+	'set-error-mode!',
+	'set-validator!',
+	'set?',
+	'short',
+	'short-array',
+	'shorts',
+	'shuffle',
+	'shutdown-agents',
+	'simple-ident?',
+	'simple-keyword?',
+	'simple-symbol?',
+	'slurp',
+	'some',
+	'some->',
+	'some->>',
+	'some-fn',
+	'some?',
+	'sort',
+	'sort-by',
+	'sorted-map',
+	'sorted-map-by',
+	'sorted-set',
+	'sorted-set-by',
+	'sorted?',
+	'special-symbol?',
+	'spit',
+	'split-at',
+	'split-with',
+	'str',
+	'string?',
+	'struct',
+	'struct-map',
+	'subs',
+	'subseq',
+	'subvec',
+	'supers',
+	'swap!',
+	'swap-vals!',
+	'symbol',
+	'symbol?',
+	'sync',
+	'tagged-literal',
+	'tagged-literal?',
+	'take',
+	'take-last',
+	'take-nth',
+	'take-while',
+	'test',
+	'the-ns',
+	'thread-bound?',
+	'time',
+	'to-array',
+	'to-array-2d',
+	'trampoline',
+	'transduce',
+	'transient',
+	'tree-seq',
+	'true?',
+	'type',
+	'unchecked-add',
+	'unchecked-add-int',
+	'unchecked-byte',
+	'unchecked-char',
+	'unchecked-dec',
+	'unchecked-dec-int',
+	'unchecked-divide-int',
+	'unchecked-double',
+	'unchecked-float',
+	'unchecked-inc',
+	'unchecked-inc-int',
+	'unchecked-int',
+	'unchecked-long',
+	'unchecked-multiply',
+	'unchecked-multiply-int',
+	'unchecked-negate',
+	'unchecked-negate-int',
+	'unchecked-remainder-int',
+	'unchecked-short',
+	'unchecked-subtract',
+	'unchecked-subtract-int',
+	'underive',
+	'unquote',
+	'unquote-splicing',
+	'unreduced',
+	'unsigned-bit-shift-right',
+	'update',
+	'update-in',
+	'update-proxy',
+	'uri?',
+	'use',
+	'uuid?',
+	'val',
+	'vals',
+	'var-get',
+	'var-set',
+	'var?',
+	'vary-meta',
+	'vec',
+	'vector',
+	'vector-of',
+	'vector?',
+	'volatile!',
+	'volatile?',
+	'vreset!',
+	'vswap!',
+	'when',
+	'when-first',
+	'when-let',
+	'when-not',
+	'when-some',
+	'while',
+	'with-bindings',
+	'with-bindings*',
+	'with-in-str',
+	'with-loading-context',
+	'with-local-vars',
+	'with-meta',
+	'with-open',
+	'with-out-str',
+	'with-precision',
+	'with-redefs',
+	'with-redefs-fn',
+	'xml-seq',
+	'zero?',
+	'zipmap',
+];
 
-	// comments
-	[
-		{
-			line: ';; comment',
-			tokens: [{ startIndex: 0, type: 'comment.clj' }],
-		},
-	],
+function createTestCases(specialForms: string[], type: string): ITestItem[] {
+	const testCases = [];
 
-	// strings
-	[
-		{
-			line: '"\\n string "',
+	for (const specialForm of specialForms) {
+		testCases.push({
+			line: `${specialForm}`,
 			tokens: [
-				{ startIndex: 0, type: 'string.clj' },
-				{ startIndex: 1, type: 'string.escape.clj' },
-				{ startIndex: 3, type: 'string.clj' },
+				{startIndex: 0, type: `${type}.clj`},
 			],
-		},
-	],
-	[
-		{
-			line: '" string \\',
-			tokens: [{ startIndex: 0, type: 'string.clj' }],
-		},
-		{
-			line: 'multiline',
-			tokens: [{ startIndex: 0, type: 'string.clj' }],
-		},
-		{
-			line: ' ',
-			tokens: [
-				// previous line needs to be terminated with \
-				{ startIndex: 0, type: 'white.clj' },
-			],
-		},
-	],
+		});
+	}
+
+	return testCases;
+}
+
+testTokenization('clojure', [
+	// special forms
+	createTestCases(specialForms, 'keyword'),
+
+	// core symbols
+	createTestCases(coreSymbols, 'keyword'),
+
+	// atoms
+	createTestCases(['false', 'nil', 'true'], 'constant'),
+
+	// keywords
+	createTestCases([':foo', '::bar', ':foo/bar', ':foo.bar/baz'], 'constant'),
 
 	// numbers
-	[
-		{
-			line: '1e2',
-			tokens: [{ startIndex: 0, type: 'number.float.clj' }],
-		},
-	],
-	[
-		{
-			line: '0xff',
-			tokens: [{ startIndex: 0, type: 'number.hex.clj' }],
-		},
-	],
+	createTestCases([
+		'42', '+42', '-421',
+		'42N', '+42N', '-42N',
+		'0.42', '+0.42', '-0.42',
+		'42M', '+42M', '-42M',
+		'42.42M', '+42.42M', '-42.42M',
+		'1/42', '+1/42', '-1/42',
+		'0x42af', '+0x42af', '-0x42af',
+		'0x42AF', '+0x42AF', '-0x42AF',
+		'1e2', '1e+2', '1e-2',
+		'+1e2', '+1e+2', '+1e-2',
+		'-1e2', '-1e+2', '-1e-2',
+		'-1.0e2', '-0.1e+2', '-1.01e-2',
+		'1E2', '1E+2', '1E-2',
+		'+1E2', '+1E+2', '+1E-2',
+		'-1E2', '-1E+2', '-1E-2',
+		'-1.0E2', '-0.1E+2', '-1.01E-2',
+		'2r101010', '+2r101010', '-2r101010',
+		'2r101010', '+2r101010', '-2r101010',
+		'8r52', '+8r52', '-8r52',
+		'36rhello', '+36rhello', '-36rhello',
+		'36rz', '+36rz', '-36rz',
+		'36rZ', '+36rZ', '-36rZ',
+	], 'number'),
+
+	// characters
+	createTestCases([
+		'\\1',
+		'\\a',
+		'\\#',
+		'\\\\',
+		'\\\"',
+		'\\(',
+		'\\A',
+		'\\backspace',
+		'\\formfeed',
+		'\\newline',
+		'\\space',
+		'\\return',
+		'\\tab',
+		'\\u1000',
+		'\\uAaAa',
+		'\\u9F9F'
+	], 'string'),
+
+	// strings
+	createTestCases([
+		'\"I\'m a little teapot.\"',
+		'\"I\'m a \\\"little\\\" teapot.\"',
+		'\"I\'m',      // this is
+		'a little',    // a multi-line
+		'teapot.\"'    // string
+	], 'string'),
+
+	// comments
+	createTestCases([
+		'; this is an in-line comment.',
+		';; this is a line comment.',
+	], 'comment'),
+
+	// reader macro characters
+	createTestCases([
+		'#',
+		'@',
+		'^',
+		'`',
+		'~',
+		"'",
+	], 'meta')
 ]);

src/clojure/clojure.ts

@@ -13,20 +13,20 @@ export const conf: IRichLanguageConfiguration = {
 		lineComment: ';;',
 	},
 
-	brackets: [['(', ')'], ['{', '}'], ['[', ']']],
+	brackets: [['(', ')'], ['[', ']'], ['{', '}']],
 
 	autoClosingPairs: [
-		{ open: '{', close: '}' },
-		{ open: '[', close: ']' },
-		{ open: '(', close: ')' },
-		{ open: '"', close: '"' },
+		{open: '(', close: ')'},
+		{open: '[', close: ']'},
+		{open: '{', close: '}'},
+		{open: '"', close: '"'},
 	],
 
 	surroundingPairs: [
-		{ open: '{', close: '}' },
-		{ open: '[', close: ']' },
-		{ open: '(', close: ')' },
-		{ open: '"', close: '"' },
+		{open: '(', close: ')'},
+		{open: '[', close: ']'},
+		{open: '{', close: '}'},
+		{open: '"', close: '"'},
 	],
 };
 
@@ -36,192 +36,740 @@ export const language = <ILanguage>{
 	tokenPostfix: '.clj',
 
 	brackets: [
-		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
-		{ open: '{', close: '}', token: 'delimiter.curly' },
-		{ open: '[', close: ']', token: 'delimiter.square' },
+		{open: '(', close: ')', token: 'delimiter.parenthesis'},
+		{open: '{', close: '}', token: 'delimiter.curly'},
+		{open: '[', close: ']', token: 'delimiter.square'},
 	],
 
-	keywords: [
-		'ns',
-		'ns-unmap',
-		'create-ns',
-		'in-ns',
-		'fn',
+	specialForms: [
+		'.',
+		'catch',
 		'def',
-		'defn',
+		'do',
+		'if',
+		'monitor-enter',
+		'monitor-exit',
+		'new',
+		'quote',
+		'recur',
+		'set!',
+		'throw',
+		'try',
+		'var',
+	],
+
+	coreSymbols: [
+		'*',
+		'*\'',
+		'*1',
+		'*2',
+		'*3',
+		'*agent*',
+		'*allow-unresolved-vars*',
+		'*assert*',
+		'*clojure-version*',
+		'*command-line-args*',
+		'*compile-files*',
+		'*compile-path*',
+		'*compiler-options*',
+		'*data-readers*',
+		'*default-data-reader-fn*',
+		'*e',
+		'*err*',
+		'*file*',
+		'*flush-on-newline*',
+		'*fn-loader*',
+		'*in*',
+		'*math-context*',
+		'*ns*',
+		'*out*',
+		'*print-dup*',
+		'*print-length*',
+		'*print-level*',
+		'*print-meta*',
+		'*print-namespace-maps*',
+		'*print-readably*',
+		'*read-eval*',
+		'*reader-resolver*',
+		'*source-path*',
+		'*suppress-read*',
+		'*unchecked-math*',
+		'*use-context-classloader*',
+		'*verbose-defrecords*',
+		'*warn-on-reflection*',
+		'+',
+		'+\'',
+		'-',
+		'-\'',
+		'->',
+		'->>',
+		'->ArrayChunk',
+		'->Eduction',
+		'->Vec',
+		'->VecNode',
+		'->VecSeq',
+		'-cache-protocol-fn',
+		'-reset-methods',
+		'..',
+		'/',
+		'<',
+		'<=',
+		'=',
+		'==',
+		'>',
+		'>=',
+		'EMPTY-NODE',
+		'Inst',
+		'StackTraceElement->vec',
+		'Throwable->map',
+		'accessor',
+		'aclone',
+		'add-classpath',
+		'add-watch',
+		'agent',
+		'agent-error',
+		'agent-errors',
+		'aget',
+		'alength',
+		'alias',
+		'all-ns',
+		'alter',
+		'alter-meta!',
+		'alter-var-root',
+		'amap',
+		'ancestors',
+		'and',
+		'any?',
+		'apply',
+		'areduce',
+		'array-map',
+		'as->',
+		'aset',
+		'aset-boolean',
+		'aset-byte',
+		'aset-char',
+		'aset-double',
+		'aset-float',
+		'aset-int',
+		'aset-long',
+		'aset-short',
+		'assert',
+		'assoc',
+		'assoc!',
+		'assoc-in',
+		'associative?',
+		'atom',
+		'await',
+		'await-for',
+		'await1',
+		'bases',
+		'bean',
+		'bigdec',
+		'bigint',
+		'biginteger',
+		'binding',
+		'bit-and',
+		'bit-and-not',
+		'bit-clear',
+		'bit-flip',
+		'bit-not',
+		'bit-or',
+		'bit-set',
+		'bit-shift-left',
+		'bit-shift-right',
+		'bit-test',
+		'bit-xor',
+		'boolean',
+		'boolean-array',
+		'boolean?',
+		'booleans',
+		'bound-fn',
+		'bound-fn*',
+		'bound?',
+		'bounded-count',
+		'butlast',
+		'byte',
+		'byte-array',
+		'bytes',
+		'bytes?',
+		'case',
+		'cast',
+		'cat',
+		'char',
+		'char-array',
+		'char-escape-string',
+		'char-name-string',
+		'char?',
+		'chars',
+		'chunk',
+		'chunk-append',
+		'chunk-buffer',
+		'chunk-cons',
+		'chunk-first',
+		'chunk-next',
+		'chunk-rest',
+		'chunked-seq?',
+		'class',
+		'class?',
+		'clear-agent-errors',
+		'clojure-version',
+		'coll?',
+		'comment',
+		'commute',
+		'comp',
+		'comparator',
+		'compare',
+		'compare-and-set!',
+		'compile',
+		'complement',
+		'completing',
+		'concat',
+		'cond',
+		'cond->',
+		'cond->>',
+		'condp',
+		'conj',
+		'conj!',
+		'cons',
+		'constantly',
+		'construct-proxy',
+		'contains?',
+		'count',
+		'counted?',
+		'create-ns',
+		'create-struct',
+		'cycle',
+		'dec',
+		'dec\'',
+		'decimal?',
+		'declare',
+		'dedupe',
+		'default-data-readers',
+		'definline',
+		'definterface',
 		'defmacro',
+		'defmethod',
 		'defmulti',
+		'defn',
+		'defn-',
 		'defonce',
-		'require',
-		'import',
-		'new',
-		'refer',
-		'pos',
-		'pos?',
+		'defprotocol',
+		'defrecord',
+		'defstruct',
+		'deftype',
+		'delay',
+		'delay?',
+		'deliver',
+		'denominator',
+		'deref',
+		'derive',
+		'descendants',
+		'destructure',
+		'disj',
+		'disj!',
+		'dissoc',
+		'dissoc!',
+		'distinct',
+		'distinct?',
+		'doall',
+		'dorun',
+		'doseq',
+		'dosync',
+		'dotimes',
+		'doto',
+		'double',
+		'double-array',
+		'double?',
+		'doubles',
+		'drop',
+		'drop-last',
+		'drop-while',
+		'eduction',
+		'empty',
+		'empty?',
+		'ensure',
+		'ensure-reduced',
+		'enumeration-seq',
+		'error-handler',
+		'error-mode',
+		'eval',
+		'even?',
+		'every-pred',
+		'every?',
+		'ex-data',
+		'ex-info',
+		'extend',
+		'extend-protocol',
+		'extend-type',
+		'extenders',
+		'extends?',
+		'false?',
+		'ffirst',
+		'file-seq',
 		'filter',
+		'filterv',
+		'find',
+		'find-keyword',
+		'find-ns',
+		'find-protocol-impl',
+		'find-protocol-method',
+		'find-var',
+		'first',
+		'flatten',
+		'float',
+		'float-array',
+		'float?',
+		'floats',
+		'flush',
+		'fn',
+		'fn?',
+		'fnext',
+		'fnil',
+		'for',
+		'force',
+		'format',
+		'frequencies',
+		'future',
+		'future-call',
+		'future-cancel',
+		'future-cancelled?',
+		'future-done?',
+		'future?',
+		'gen-class',
+		'gen-interface',
+		'gensym',
+		'get',
+		'get-in',
+		'get-method',
+		'get-proxy-class',
+		'get-thread-bindings',
+		'get-validator',
+		'group-by',
+		'halt-when',
+		'hash',
+		'hash-combine',
+		'hash-map',
+		'hash-ordered-coll',
+		'hash-set',
+		'hash-unordered-coll',
+		'ident?',
+		'identical?',
+		'identity',
+		'if-let',
+		'if-not',
+		'if-some',
+		'ifn?',
+		'import',
+		'in-ns',
+		'inc',
+		'inc\'',
+		'indexed?',
+		'init-proxy',
+		'inst-ms',
+		'inst-ms*',
+		'inst?',
+		'instance?',
+		'int',
+		'int-array',
+		'int?',
+		'integer?',
+		'interleave',
+		'intern',
+		'interpose',
+		'into',
+		'into-array',
+		'ints',
+		'io!',
+		'isa?',
+		'iterate',
+		'iterator-seq',
+		'juxt',
+		'keep',
+		'keep-indexed',
+		'key',
+		'keys',
+		'keyword',
+		'keyword?',
+		'last',
+		'lazy-cat',
+		'lazy-seq',
+		'let',
+		'letfn',
+		'line-seq',
+		'list',
+		'list*',
+		'list?',
+		'load',
+		'load-file',
+		'load-reader',
+		'load-string',
+		'loaded-libs',
+		'locking',
+		'long',
+		'long-array',
+		'longs',
+		'loop',
+		'macroexpand',
+		'macroexpand-1',
+		'make-array',
+		'make-hierarchy',
 		'map',
+		'map-entry?',
+		'map-indexed',
+		'map?',
+		'mapcat',
+		'mapv',
+		'max',
+		'max-key',
+		'memfn',
+		'memoize',
+		'merge',
+		'merge-with',
+		'meta',
+		'method-sig',
+		'methods',
+		'min',
+		'min-key',
+		'mix-collection-hash',
+		'mod',
+		'munge',
+		'name',
+		'namespace',
+		'namespace-munge',
+		'nat-int?',
+		'neg-int?',
+		'neg?',
+		'newline',
+		'next',
+		'nfirst',
+		'nil?',
+		'nnext',
+		'not',
+		'not-any?',
+		'not-empty',
+		'not-every?',
+		'not=',
+		'ns',
+		'ns-aliases',
+		'ns-imports',
+		'ns-interns',
+		'ns-map',
+		'ns-name',
+		'ns-publics',
+		'ns-refers',
+		'ns-resolve',
+		'ns-unalias',
+		'ns-unmap',
+		'nth',
+		'nthnext',
+		'nthrest',
+		'num',
+		'number?',
+		'numerator',
+		'object-array',
+		'odd?',
+		'or',
+		'parents',
+		'partial',
+		'partition',
+		'partition-all',
+		'partition-by',
+		'pcalls',
+		'peek',
+		'persistent!',
+		'pmap',
+		'pop',
+		'pop!',
+		'pop-thread-bindings',
+		'pos-int?',
+		'pos?',
+		'pr',
+		'pr-str',
+		'prefer-method',
+		'prefers',
+		'primitives-classnames',
+		'print',
+		'print-ctor',
+		'print-dup',
+		'print-method',
+		'print-simple',
+		'print-str',
+		'printf',
+		'println',
+		'println-str',
+		'prn',
+		'prn-str',
+		'promise',
+		'proxy',
+		'proxy-call-with-super',
+		'proxy-mappings',
+		'proxy-name',
+		'proxy-super',
+		'push-thread-bindings',
+		'pvalues',
+		'qualified-ident?',
+		'qualified-keyword?',
+		'qualified-symbol?',
+		'quot',
+		'rand',
+		'rand-int',
+		'rand-nth',
+		'random-sample',
+		'range',
+		'ratio?',
+		'rational?',
+		'rationalize',
+		're-find',
+		're-groups',
+		're-matcher',
+		're-matches',
+		're-pattern',
+		're-seq',
+		'read',
+		'read-line',
+		'read-string',
+		'reader-conditional',
+		'reader-conditional?',
+		'realized?',
+		'record?',
 		'reduce',
+		'reduce-kv',
+		'reduced',
+		'reduced?',
+		'reductions',
+		'ref',
+		'ref-history-count',
+		'ref-max-history',
+		'ref-min-history',
+		'ref-set',
+		'refer',
+		'refer-clojure',
+		'reify',
+		'release-pending-sends',
+		'rem',
+		'remove',
+		'remove-all-methods',
+		'remove-method',
+		'remove-ns',
+		'remove-watch',
 		'repeat',
-		'key',
+		'repeatedly',
+		'replace',
+		'replicate',
+		'require',
+		'reset!',
+		'reset-meta!',
+		'reset-vals!',
+		'resolve',
 		'rest',
-		'concat',
-		'into',
+		'restart-agent',
+		'resultset-seq',
 		'reverse',
-		'iterate',
-		'range',
-		'drop',
-		'drop-while',
+		'reversible?',
+		'rseq',
+		'rsubseq',
+		'run!',
+		'satisfies?',
+		'second',
+		'select-keys',
+		'send',
+		'send-off',
+		'send-via',
+		'seq',
+		'seq?',
+		'seqable?',
+		'seque',
+		'sequence',
+		'sequential?',
+		'set',
+		'set-agent-send-executor!',
+		'set-agent-send-off-executor!',
+		'set-error-handler!',
+		'set-error-mode!',
+		'set-validator!',
+		'set?',
+		'short',
+		'short-array',
+		'shorts',
+		'shuffle',
+		'shutdown-agents',
+		'simple-ident?',
+		'simple-keyword?',
+		'simple-symbol?',
+		'slurp',
+		'some',
+		'some->',
+		'some->>',
+		'some-fn',
+		'some?',
+		'sort',
+		'sort-by',
+		'sorted-map',
+		'sorted-map-by',
+		'sorted-set',
+		'sorted-set-by',
+		'sorted?',
+		'special-symbol?',
+		'spit',
+		'split-at',
+		'split-with',
+		'str',
+		'string?',
+		'struct',
+		'struct-map',
+		'subs',
+		'subseq',
+		'subvec',
+		'supers',
+		'swap!',
+		'swap-vals!',
+		'symbol',
+		'symbol?',
+		'sync',
+		'tagged-literal',
+		'tagged-literal?',
 		'take',
+		'take-last',
+		'take-nth',
 		'take-while',
-		'neg',
-		'neg?',
-		'bound-fn',
-		'if',
-		'if-not',
-		'if-let',
-		'case,',
-		'contains',
-		'conj',
-		'disj',
-		'sort',
-		'get',
-		'assoc',
-		'merge',
-		'keys',
+		'test',
+		'the-ns',
+		'thread-bound?',
+		'time',
+		'to-array',
+		'to-array-2d',
+		'trampoline',
+		'transduce',
+		'transient',
+		'tree-seq',
+		'true?',
+		'type',
+		'unchecked-add',
+		'unchecked-add-int',
+		'unchecked-byte',
+		'unchecked-char',
+		'unchecked-dec',
+		'unchecked-dec-int',
+		'unchecked-divide-int',
+		'unchecked-double',
+		'unchecked-float',
+		'unchecked-inc',
+		'unchecked-inc-int',
+		'unchecked-int',
+		'unchecked-long',
+		'unchecked-multiply',
+		'unchecked-multiply-int',
+		'unchecked-negate',
+		'unchecked-negate-int',
+		'unchecked-remainder-int',
+		'unchecked-short',
+		'unchecked-subtract',
+		'unchecked-subtract-int',
+		'underive',
+		'unquote',
+		'unquote-splicing',
+		'unreduced',
+		'unsigned-bit-shift-right',
+		'update',
+		'update-in',
+		'update-proxy',
+		'uri?',
+		'use',
+		'uuid?',
+		'val',
 		'vals',
-		'nth',
-		'first',
-		'last',
-		'count',
-		'contains?',
-		'cond',
-		'condp',
-		'cond->',
-		'cond->>',
+		'var-get',
+		'var-set',
+		'var?',
+		'vary-meta',
+		'vec',
+		'vector',
+		'vector-of',
+		'vector?',
+		'volatile!',
+		'volatile?',
+		'vreset!',
+		'vswap!',
 		'when',
-		'while',
-		'when-not',
-		'when-let',
 		'when-first',
-		'do',
-		'future',
-		'comment',
-		'doto',
-		'locking',
-		'proxy',
-		'println',
-		'type',
-		'meta',
-		'var',
-		'as->',
-		'reify',
-		'deftype',
-		'defrecord',
-		'defprotocol',
-		'extend',
-		'extend-protocol',
-		'extend-type',
-		'specify',
-		'specify!',
-		'try',
-		'catch',
-		'finally',
-		'let',
-		'letfn',
-		'binding',
-		'loop',
-		'for',
-		'seq',
-		'doseq',
-		'dotimes',
 		'when-let',
-		'if-let',
+		'when-not',
 		'when-some',
-		'if-some',
-		'this-as',
-		'defmethod',
-		'testing',
-		'deftest',
-		'are',
-		'use-fixtures',
-		'use',
-		'remove',
-		'run',
-		'run*',
-		'fresh',
-		'alt!',
-		'alt!!',
-		'go',
-		'go-loop',
-		'thread',
-		'boolean',
-		'str',
+		'while',
+		'with-bindings',
+		'with-bindings*',
+		'with-in-str',
+		'with-loading-context',
+		'with-local-vars',
+		'with-meta',
+		'with-open',
+		'with-out-str',
+		'with-precision',
+		'with-redefs',
+		'with-redefs-fn',
+		'xml-seq',
+		'zero?',
+		'zipmap',
 	],
 
 	constants: ['true', 'false', 'nil'],
 
-	operators: [
-		'=',
-		'not=',
-		'<',
-		'<=',
-		'>',
-		'>=',
-		'and',
-		'or',
-		'not',
-		'inc',
-		'dec',
-		'max',
-		'min',
-		'rem',
-		'bit-and',
-		'bit-or',
-		'bit-xor',
-		'bit-not',
-	],
+	symbolCharacter: /[!#'*+\-.\/:<=>?_\w\xa1-\uffff]/,
+
+	numbers: /^[+\-]?\d+(?:(?:N|(?:[eE][+\-]?\d+))|(?:\.?\d*(?:M|(?:[eE][+\-]?\d+))?)|\/\d+|[xX][0-9a-fA-F]+|r[0-9a-zA-Z]+)?/,
+
+	characters: /\\(?:@symbolCharacter+|[\\"()\[\]{}]|x[0-9A-Fa-f]{4}|u[0-9A-Fa-f]{4}|o[0-7]{3})/,
 
 	tokenizer: {
 		root: [
-			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
-			[/[+-]?\d+(?:(?:\.\d*)?(?:[eE][+-]?\d+)?)?/, 'number.float'],
+			// numbers
+			[/@numbers/, 'number'],
 
-			[
-				/(?:\b(?:(ns|def|defn|defn-|defmacro|defmulti|defonce|ns|ns-unmap|fn))(?![\w-]))(\s+)((?:\w|\-|\!|\?)*)/,
-				['keyword', 'white', 'variable'],
-			],
+			// characters
+			[/@characters/, 'string'],
 
-			[
-				/[a-zA-Z_#][a-zA-Z0-9_\-\?\!\*]*/,
-				{
-					cases: {
-						'@keywords': 'keyword',
-						'@constants': 'constant',
-						'@operators': 'operators',
-						'@default': 'identifier',
-					},
+			// brackets
+			[/[()\[\]{}]/, '@brackets'],
+
+			// regular expressions
+			[/\/#"(?:\.|(?:")|[^"\n])*"\/g/, 'regexp'],
+
+			// inline comments
+			[/;.*$/, 'comment'],
+
+			// reader macro characters
+			[/[#'@^`~]/, 'meta'],
+
+			// keywords
+			[/:@symbolCharacter+/, 'constant'],
+
+			// symbols
+			[/@symbolCharacter+/, {
+				cases: {
+					'@specialForms': 'keyword',
+					'@coreSymbols': 'keyword',
+					'@constants': 'constant',
+					'@default': 'identifier',
 				},
+			},
 			],
 
-			[/\/#"(?:\.|(?:\")|[^""\n])*"\/g/, 'regexp'],
-
-			{ include: '@whitespace' },
-			{ include: '@strings' },
+			{include: '@whitespace'},
+			{include: '@string'},
 		],
 
-		whitespace: [[/[ \t\r\n]+/, 'white'], [/;;.*$/, 'comment']],
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+			[/;;.*$/, 'comment']],
 
-		strings: [
-			[/"$/, 'string', '@popall'],
-			[/"(?=.)/, 'string', '@multiLineString'],
+		string: [
+			[/"/, 'string', '@multiLineString'],
 		],
 
 		multiLineString: [
-			[/\\./, 'string.escape'],
-			[/"/, 'string', '@popall'],
-			[/.(?=.*")/, 'string'],
-			[/.*\\$/, 'string'],
-			[/.*$/, 'string', '@popall'],
+			[/[^\\"$]+/, 'string'],
+			[/@characters/, 'string'],
+			[/"/, 'string', '@pop']
 		],
 	},
 };