added class and member detection

README.md

@@ -27,6 +27,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed
 * swift
 * vb
 * xml
+* small basic
 
 Also `css` dialects:
 

src/sb.ts

@@ -43,44 +43,46 @@ export const language = <ILanguage>{
 	],
 
 	keywords: [
-		'And', 'Else', 'ElseIf', 'EndFor', 'EndIf', 'EndSub', 'EndWhile',
-		'For', 'Goto', 'If', 'Or', 'Step', 'Sub', 'Then', 'To', 'While'
+		'Else', 'ElseIf', 'EndFor', 'EndIf', 'EndSub', 'EndWhile',
+		'For', 'Goto', 'If', 'Step', 'Sub', 'Then', 'To', 'While'
 	],
 
 	tagwords: [
 		'If', 'Sub', 'While', 'For'
 	],
 
-	operators: [
-		'>', '<', '<>', '<=', '>=', 'And',
-		'Or', '+', '-', '*', '/', '=',
-	],
+	operators: [ '>', '<', '<>', '<=', '>=', 'And', 'Or', '+', '-', '*', '/', '=' ],
 
 	// we include these common regular expressions
-	symbols: /[=><.,:+\-\/]+/,
+	identifier: /[a-zA-Z_][\w]*/,
+	symbols: /[=><:+\-*\/%\.,]+/,
 	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
 
 	// The main tokenizer for our languages
 	tokenizer: {
 		root: [
-
 			// whitespace
 			{ include: '@whitespace' },
 
-			// usual ending tags
-			[/end([a-zA-Z_]\w*)/, { token: 'keyword.tag-$1' }],
+            // classes
+            [/(@identifier)(?=[.])/, 'type'],
 
 			// identifiers, tagwords, and keywords
-			[/[a-zA-Z_]\w*/, {
+			[/@identifier/, {
 				cases: {
-					'@tagwords': { token: 'keyword.tag-$0' },
 					'@keywords': { token: 'keyword.$0' },
-					'@default': 'identifier'
+					'@operators': 'operator',
+					'@default': 'variable.name'
 				}
 			}],
 
-			// Preprocessor directive
-			[/^\s*#\w+/, 'keyword'],
+            // methods, properties, and events
+			[/([.])(@identifier)/, {
+				cases: {
+					'$2': ['delimiter', 'type.member'],
+					'@default': ''
+				}
+			}],
 
 			// numbers
 			[/\d*\.\d+/, 'number.float'],
@@ -88,7 +90,12 @@ export const language = <ILanguage>{
 
 			// delimiters and operators
 			[/[()\[\]]/, '@brackets'],
-			[/@symbols/, 'delimiter'],
+			[/@symbols/, {
+				cases: {
+					'@operators': 'operator',
+					'@default': 'delimiter'
+				}
+			}],
 
 			// strings
 			[/"([^"\\]|\\.)*$/, 'string.invalid'],  // non-teminated string

test/sb.test.ts

@@ -9,7 +9,7 @@ import { testTokenization } from './testRunner';
 
 testTokenization('sb', [
 
-	// Comments - single line
+	// Comments
 	[{
 		line: '\'',
 		tokens: [
@@ -49,15 +49,13 @@ testTokenization('sb', [
 	}],
 
 	[{
-		line: 'x = 1 \' my comment \'\' is a nice one',
+		line: 'x=1 \' my comment \'\' is a nice one',
 		tokens: [
-			{ startIndex: 0, type: 'identifier.sb' },
-			{ startIndex: 1, type: '' },
-			{ startIndex: 2, type: 'delimiter.sb' },
+			{ startIndex: 0, type: 'variable.name.sb' }, // x
+			{ startIndex: 1, type: 'operator.sb' }, // =
+			{ startIndex: 2, type: 'number.sb' }, // 1
 			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'number.sb' },
-			{ startIndex: 5, type: '' },
-			{ startIndex: 6, type: 'comment.sb' }
+			{ startIndex: 4, type: 'comment.sb' }
 		]
 	}],
 
@@ -88,7 +86,7 @@ testTokenization('sb', [
 		tokens: [
 			{ startIndex: 0, type: 'number.sb' },
 			{ startIndex: 1, type: '' },
-			{ startIndex: 2, type: 'delimiter.sb' },
+			{ startIndex: 2, type: 'operator.sb' },
 			{ startIndex: 3, type: 'number.sb' }
 		]
 	}],
@@ -97,7 +95,7 @@ testTokenization('sb', [
 		line: '100+10',
 		tokens: [
 			{ startIndex: 0, type: 'number.sb' },
-			{ startIndex: 3, type: 'delimiter.sb' },
+			{ startIndex: 3, type: 'operator.sb' },
 			{ startIndex: 4, type: 'number.sb' }
 		]
 	}],
@@ -107,7 +105,7 @@ testTokenization('sb', [
 		tokens: [
 			{ startIndex: 0, type: 'number.sb' },
 			{ startIndex: 1, type: '' },
-			{ startIndex: 2, type: 'delimiter.sb' },
+			{ startIndex: 2, type: 'operator.sb' },
 			{ startIndex: 3, type: '' },
 			{ startIndex: 4, type: 'number.sb' }
 		]
@@ -115,11 +113,65 @@ testTokenization('sb', [
 
 	// Keywords
 	[{
-		line: 'Sub Foo',
+		line: 'For i = 0 To 10 Step 2',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-sub.sb' },
+			{ startIndex: 0, type: 'keyword.for.sb' }, // For
 			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'identifier.sb' }
+			{ startIndex: 4, type: 'variable.name.sb' }, // i
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'operator.sb' }, // =
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.sb' }, // 0
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'keyword.to.sb' }, // To
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'number.sb' }, // 10
+			{ startIndex: 15, type: '' },
+			{ startIndex: 16, type: 'keyword.step.sb' }, // Step
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'number.sb' } // 2
+		]
+	}],
+
+	[{
+		line: 'if x <> 23.7 thEn',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.if.sb' }, // if
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'variable.name.sb' }, // x
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'operator.sb' }, // <>
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'number.float.sb' }, // 23.7
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'keyword.then.sb' } // thEn
+		]
+	}],
+
+	[{
+		line: 'ElseIf b[i] And col Then',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.elseif.sb' }, // ElseIf
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'variable.name.sb' }, // b
+			{ startIndex: 8, type: 'delimiter.array.sb' }, // [
+			{ startIndex: 9, type: 'variable.name.sb' }, // i
+			{ startIndex: 10, type: 'delimiter.array.sb' }, // ]
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'operator.sb' }, // And
+			{ startIndex: 15, type: '' },
+			{ startIndex: 16, type: 'variable.name.sb' }, // col
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'keyword.then.sb' } // Then
+		]
+	}],
+
+	[{
+		line: 'GoTo location',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.goto.sb' }, // GoTo
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'variable.name.sb' }, // location
 		]
 	}],
 
@@ -127,19 +179,19 @@ testTokenization('sb', [
 	[{
 		line: 's = "string"',
 		tokens: [
-			{ startIndex: 0, type: 'identifier.sb' },
+			{ startIndex: 0, type: 'variable.name.sb' }, // s
 			{ startIndex: 1, type: '' },
-			{ startIndex: 2, type: 'delimiter.sb' },
+			{ startIndex: 2, type: 'operator.sb' }, // =
 			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'string.sb' }
+			{ startIndex: 4, type: 'string.sb' } // "string"
 		]
 	}],
 
 	[{
 		line: '"use strict";',
 		tokens: [
-			{ startIndex: 0, type: 'string.sb' },
-			{ startIndex: 12, type: '' }
+			{ startIndex: 0, type: 'string.sb' }, // "use strict"
+			{ startIndex: 12, type: '' } // ;
 		]
 	}],
 
@@ -147,110 +199,89 @@ testTokenization('sb', [
 	[{
 		line: 'sub ToString',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-sub.sb' },
+			{ startIndex: 0, type: 'keyword.sub.sb' }, // sub
 			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'identifier.sb' }
+			{ startIndex: 4, type: 'variable.name.sb' } // ToString
 		]
 	}],
 
 	[{
 		line: 'While For If Sub EndWhile EndFor EndIf endsub',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-while.sb' },
+			{ startIndex: 0, type: 'keyword.while.sb' }, // While
 			{ startIndex: 5, type: '' },
-			{ startIndex: 6, type: 'keyword.tag-for.sb' },
+			{ startIndex: 6, type: 'keyword.for.sb' }, // For
 			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: 'keyword.tag-if.sb' },
+			{ startIndex: 10, type: 'keyword.if.sb' }, // If
 			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: 'keyword.tag-sub.sb' },
+			{ startIndex: 13, type: 'keyword.sub.sb' }, // Sub
 			{ startIndex: 16, type: '' },
-			{ startIndex: 17, type: 'keyword.tag-while.sb' },
+			{ startIndex: 17, type: 'keyword.endwhile.sb' }, // EndWhile
 			{ startIndex: 25, type: '' },
-			{ startIndex: 26, type: 'keyword.tag-for.sb' },
+			{ startIndex: 26, type: 'keyword.endfor.sb' }, // EndFor
 			{ startIndex: 32, type: '' },
-			{ startIndex: 33, type: 'keyword.tag-if.sb' },
+			{ startIndex: 33, type: 'keyword.endif.sb' }, // EndIf
 			{ startIndex: 38, type: '' },
-			{ startIndex: 39, type: 'keyword.tag-sub.sb' }
+			{ startIndex: 39, type: 'keyword.endsub.sb' } //endsub
 		]
 	}],
 
 	[{
 		line: 'While while WHILE WHile whiLe',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-while.sb' },
+			{ startIndex: 0, type: 'keyword.while.sb' }, // While
 			{ startIndex: 5, type: '' },
-			{ startIndex: 6, type: 'keyword.tag-while.sb' },
+			{ startIndex: 6, type: 'keyword.while.sb' }, // while
 			{ startIndex: 11, type: '' },
-			{ startIndex: 12, type: 'keyword.tag-while.sb' },
+			{ startIndex: 12, type: 'keyword.while.sb' }, // WHILE
 			{ startIndex: 17, type: '' },
-			{ startIndex: 18, type: 'keyword.tag-while.sb' },
+			{ startIndex: 18, type: 'keyword.while.sb' }, // WHile
 			{ startIndex: 23, type: '' },
-			{ startIndex: 24, type: 'keyword.tag-while.sb' }
+			{ startIndex: 24, type: 'keyword.while.sb' } // whiLe
 		]
 	}],
 
+	// types and members
 	[{
-		line: 'If b(i) = col Then',
+		line: 'Else TextWindow.Write("text")',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-if.sb' },
-			{ startIndex: 2, type: '' },
-			{ startIndex: 3, type: 'identifier.sb' },
-			{ startIndex: 4, type: 'delimiter.parenthesis.sb' },
-			{ startIndex: 5, type: 'identifier.sb' },
-			{ startIndex: 6, type: 'delimiter.parenthesis.sb' },
-			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'delimiter.sb' },
-			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: 'identifier.sb' },
-			{ startIndex: 13, type: '' },
-			{ startIndex: 14, type: 'keyword.then.sb' }
+			{ startIndex: 0, type: 'keyword.else.sb' }, // Else
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'type.sb' }, // TextWindow
+			{ startIndex: 15, type: 'delimiter.sb' }, // .
+			{ startIndex: 16, type: 'type.member.sb' }, // Write
+			{ startIndex: 21, type: 'delimiter.parenthesis.sb' }, // (
+			{ startIndex: 22, type: 'string.sb' }, // "text"
+			{ startIndex: 28, type: 'delimiter.parenthesis.sb' } // )
 		]
 	}],
 
 	[{
-		line: 'For i = 0 To 10 Step 2 DoStuff EndFor',
+		line: 'class.method (x, y)',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-for.sb' },
-			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'identifier.sb' },
-			{ startIndex: 5, type: '' },
-			{ startIndex: 6, type: 'delimiter.sb' },
-			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'number.sb' },
-			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: 'keyword.to.sb' },
+			{ startIndex: 0, type: 'type.sb' }, // class
+			{ startIndex: 5, type: 'delimiter.sb' }, // .
+			{ startIndex: 6, type: 'type.member.sb' }, // method
 			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: 'number.sb' },
-			{ startIndex: 15, type: '' },
-			{ startIndex: 16, type: 'keyword.step.sb' },
-			{ startIndex: 20, type: '' },
-			{ startIndex: 21, type: 'number.sb' },
-			{ startIndex: 22, type: '' },
-			{ startIndex: 23, type: 'identifier.sb' },
-			{ startIndex: 30, type: '' },
-			{ startIndex: 31, type: 'keyword.tag-for.sb' }
-		]
-	}],
-
-	[{
-		line: 'For stuff EndFor',
-		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-for.sb' },
-			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'identifier.sb' },
-			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: 'keyword.tag-for.sb' },
+			{ startIndex: 13, type: 'delimiter.parenthesis.sb' }, // (
+			{ startIndex: 14, type: 'variable.name.sb' }, // x
+			{ startIndex: 15, type: 'delimiter.sb' }, // ,
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'variable.name.sb' }, // y
+			{ startIndex: 18, type: 'delimiter.parenthesis.sb' } // )
 		]
 	}],
 
 	[{
-		line: 'for stuff endfor',
+		line: 'const = Math.PI',
 		tokens: [
-			{ startIndex: 0, type: 'keyword.tag-for.sb' },
-			{ startIndex: 3, type: '' },
-			{ startIndex: 4, type: 'identifier.sb' },
-			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: 'keyword.tag-for.sb' },
+			{ startIndex: 0, type: 'variable.name.sb' }, // const
+			{ startIndex: 5, type: '' },
+			{ startIndex: 6, type: 'operator.sb' }, // =
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'type.sb' }, // Math
+			{ startIndex: 12, type: 'delimiter.sb' }, // .
+			{ startIndex: 13, type: 'type.member.sb' } // PI
 		]
 	}]
 ]);