add support for reStructuredText

src/monaco.contribution.ts

@@ -63,3 +63,4 @@ import './typescript/typescript.contribution';
 import './vb/vb.contribution';
 import './xml/xml.contribution';
 import './yaml/yaml.contribution';
+import './rst/restructuredtext.contribution';

src/rst/restructuredtext.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'restructuredtext',
+	extensions: ['.rst'],
+	aliases: ['reStructuredText', 'restructuredtext'],
+	loader: () => import('./restructuredtext')
+});

src/rst/restructuredtext.test.ts

@@ -0,0 +1,492 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('restructuredtext', [
+	[
+		{
+			line: '#####',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' }
+			]
+		}
+	],
+	[
+		{
+			line: 'strong **strong**',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 7, type: 'strong.rst' },
+			]
+		},
+		{
+			line: 'emphasis *emphasis*',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 9, type: 'emphasis.rst' },
+			]
+		}
+	],
+	[
+		{
+			line: '.. [23] This is the footnote',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 6, type: '' },
+			]
+		},
+		{
+			line: '.. [#ab] This is the footnote',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 7, type: '' },
+			]
+		},
+		{
+			line: '.. [#] This is the footnote',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 5, type: '' },
+			]
+		},
+		{
+			line: '.. [*] This is the footnote',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 5, type: '' },
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: ' .. [23] This is not the footnote',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '[23]_',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'string.link.rst' },
+				{ startIndex: 3, type: '' }
+
+			]
+		},
+		{
+			line: '[*]_',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'string.link.rst' },
+				{ startIndex: 2, type: '' }
+			]
+		},
+		{
+			line: '[#]_',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'string.link.rst' },
+				{ startIndex: 2, type: '' }
+			]
+		},
+		{
+			line: '[#abc]_ [#]_',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'string.link.rst' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 9, type: 'string.link.rst' },
+				{ startIndex: 10, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '.. [A3] This is the citation',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 6, type: '' }
+			]
+		}
+	],
+	[
+		{
+			line: ' .. [A3] This is not the citation',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '.. [A3] This is the citation',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 6, type: '' },
+			]
+		},
+		{
+			line: ' first line',
+			tokens: [
+				{ startIndex: 0, type: '' }
+			]
+		},
+		{
+			line: ' second line',
+			tokens: [
+				{ startIndex: 0, type: '' }
+			]
+		},
+		{
+			line: 'new line starts',
+			tokens: [
+				{ startIndex: 0, type: '' }
+			]
+		}
+	],
+	[
+		{
+			line: '[A3]_',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'string.link.rst' },
+				{ startIndex: 3, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: 'Interpreted Text `text`',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		},
+		{
+			line: ' .. _`text`: paragraph',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 6, type: 'string.link.rst' },
+				{ startIndex: 10, type: '' },
+			]
+		},
+		{
+			line: 'Interpreted Text :role:`text`',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 18, type: 'keyword.rst' },
+				{ startIndex: 22, type: '' },
+			]
+		},
+		{
+			line: 'Interpreted Text `text`:role:',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 24, type: 'keyword.rst' },
+				{ startIndex: 28, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '.. note:: This is a directive',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.rst' },
+				{ startIndex: 7, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: 'link .. _link: this is not a hyperlink',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		},
+		{
+			line: '.. _link: this is a hyperlink',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 10, type: 'string.link.rst' },
+			]
+		},
+		{
+			line: '.. _`link`: this is a hyperlink',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 12, type: 'string.link.rst' },
+			]
+		},
+		{
+			line: '.. __: this is a anonymous hyperlink',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 7, type: 'string.link.rst' },
+			]
+		},
+		{
+			line: '__: this is a anonymous hyperlink',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+			]
+		}
+	],
+	[
+		{
+			line: '...... _`this is a inline internal target`',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 9, type: 'string.link.rst' },
+				{ startIndex: 41, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '.. |biohazard| image:: biohazard.png',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'string.link.rst' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 15, type: 'keyword.rst' },
+				{ startIndex: 20, type: '' },
+			]
+		},
+		{
+			line: '... |biohazard| ...',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 5, type: 'string.link.rst' },
+				{ startIndex: 14, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: 'ref_ `ref`_ ref__ `ref`__',
+			tokens: [
+				{ startIndex: 0, type: 'string.link.rst' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 5, type: 'string.link.rst' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 12, type: 'string.link.rst' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 18, type: 'string.link.rst' },
+				{ startIndex: 23, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '.... `title <http://google.com>`_',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 6, type: 'string.link.rst' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'string.link.rst' },
+				{ startIndex: 30, type: '' },
+			]
+		}
+	],
+	[
+		{
+			line: '::',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+			]
+		},
+		{
+			line: '',
+			tokens: [
+			]
+		},
+		{
+			line: ' first line',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 1, type: '' },
+			]
+		},
+		{
+			line: ' second line',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 1, type: '' },
+			]
+		},
+		{
+			line: '',
+			tokens: [
+			]
+		},
+		{
+			line: ' paragraph',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		},
+	],
+	[
+		{
+			line: 'desc ::',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'keyword.rst' },
+			]
+		},
+		{
+			line: '',
+			tokens: [
+			]
+		},
+		{
+			line: '>>first line',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 2, type: '' },
+			]
+		},
+		{
+			line: '>',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+			]
+		},
+		{
+			line: '>second line',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 1, type: '' },
+			]
+		},
+		{
+			line: '',
+			tokens: [
+			]
+		},
+		{
+			line: ' paragraph',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		},
+	],
+	[
+		{
+			line: '.. comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'comment.rst' },
+			]
+		},
+		{
+			line: ' firstline',
+			tokens: [
+				{ startIndex: 0, type: 'comment.rst' },
+			]
+		},
+		{
+			line: '',
+			tokens: [
+			]
+		},
+		{
+			line: ' paragraph',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			]
+		},
+	],
+	[
+		{
+			line: '==',
+			tokens: [
+				{ startIndex: 0, type: '' },
+			],
+		},
+		{
+			line: '===',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+			],
+		},
+	],
+	[
+		{
+			line: '1. item',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 3, type: '' },
+			],
+		},
+		{
+			line: 'a. item',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 3, type: '' },
+			],
+		},
+		{
+			line: '* item',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 2, type: '' },
+			],
+		},
+		{
+			line: '- item',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 2, type: '' },
+			],
+		},
+		{
+			line: '1) item',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 3, type: '' },
+			],
+		},
+		{
+			line: '(a) item',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+				{ startIndex: 4, type: '' },
+			],
+		},
+	],
+	[
+		{
+			line: '+------------------------+------------+----------+----------+',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+			],
+		},
+		{
+			line: '+========================+============+==========+==========+',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.rst' },
+			],
+		},
+	],
+]);

src/rst/restructuredtext.ts

@@ -0,0 +1,169 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>', notIn: ['string'] }
+	],
+	surroundingPairs: [
+		{ open: '(', close: ')' },
+		{ open: '[', close: ']' },
+		{ open: '`', close: '`' },
+	],
+	folding: {
+		markers: {
+			start: new RegExp("^\\s*<!--\\s*#?region\\b.*-->"),
+			end: new RegExp("^\\s*<!--\\s*#?endregion\\b.*-->")
+		}
+	}
+};
+
+export const language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.rst',
+
+	control: /[\\`*_\[\]{}()#+\-\.!]/,
+	escapes: /\\(?:@control)/,
+
+	empty: [
+		'area', 'base', 'basefont', 'br', 'col', 'frame',
+		'hr', 'img', 'input', 'isindex', 'link', 'meta', 'param'
+	],
+
+	alphanumerics: /[A-Za-z0-9]/,
+	alphanumericsplus: /[A-Za-z0-9-_+:.]/,
+	simpleRefNameWithoutBq: /(?:@alphanumerics@alphanumericsplus*@alphanumerics)+|(?:@alphanumerics+)/,
+	simpleRefName: /(?:`@simpleRefNameWithoutBq`|@simpleRefNameWithoutBq)/,
+	phrase: /@simpleRefName(?:\s@simpleRefName)*/,
+	citationName: /[A-Za-z][A-Za-z0-9-_.]*/,
+	blockLiteralStart: /(?:[!"#$%&'()*+,-./:;<=>?@\[\]^_`{|}~]|[\s])/,
+	precedingChars: /(?:[ -:/'"<([{])/,
+	followingChars: /(?:[ -.,:;!?/'")\]}>]|$)/,
+	punctuation: /(=|-|~|`|#|"|\^|\+|\*|:|\.|'|_|\+)/,
+	tokenizer: {
+		root: [
+			//sections
+			[/^(@punctuation{3,}$){1,1}?/, 'keyword'],
+
+			//line-blocks
+			//No rules on it
+
+			//bullet-lists
+			[/^\s*([\*\-+‣•]|[a-zA-Z0-9]+\.|\([a-zA-Z0-9]+\)|[a-zA-Z0-9]+\))\s/, 'keyword'],
+
+			//literal-blocks
+			[/([ ]::)\s*$/, 'keyword', '@blankLineOfLiteralBlocks'],
+			[/(::)\s*$/, 'keyword', '@blankLineOfLiteralBlocks'],
+
+			{ include: '@tables' },
+
+			{ include: '@explicitMarkupBlocks' },
+
+			{ include: '@inlineMarkup' },
+		],
+		explicitMarkupBlocks: [
+			//citations
+			{ include: '@citations' },
+			//footnotes
+			{ include: '@footnotes' },
+			//directives
+			[/^(\.\.\s)(@simpleRefName)(::\s)(.*)$/, [{ token: '', next: 'subsequentLines' }, 'keyword', '', '']],
+
+			//hyperlink-targets
+			[/^(\.\.)(\s+)(_)(@simpleRefName)(:)(\s+)(.*)/, [{ token: '', next: 'hyperlinks' }, '', '', 'string.link', '', '', 'string.link']],
+
+			//anonymous-hyperlinks
+			[/^((?:(?:\.\.)(?:\s+))?)(__)(:)(\s+)(.*)/, [{ token: '', next: 'subsequentLines' }, '', '', '', 'string.link']],
+			[/^(__\s+)(.+)/, ['', 'string.link']],
+
+			//substitution-definitions
+			[/^(\.\.)( \|)([^| ]+[^|]*[^| ]*)(\| )(@simpleRefName)(:: .*)/, [{ token: '', next: 'subsequentLines' }, '', 'string.link', '', 'keyword', ''], '@rawBlocks'],
+			[/(\|)([^| ]+[^|]*[^| ]*)(\|_{0,2})/, ['', 'string.link', '']],
+
+			//comments
+			[/^(\.\.)([ ].*)$/, [{ token: '', next: '@comments' }, 'comment']],
+		],
+		inlineMarkup: [
+			{ include: '@citationsReference' },
+			{ include: '@footnotesReference' },
+
+			//hyperlink-references
+			[/(@simpleRefName)(_{1,2})/, ['string.link', '']],
+
+			//embedded-uris-and-aliases
+			[/(`)([^<`]+\s+)(<)(.*)(>)(`)(_)/, ['', 'string.link', '', 'string.link', '', '', '']],
+
+			//emphasis
+			[/\*\*([^\\*]|\*(?!\*))+\*\*/, 'strong'],
+			[/\*[^*]+\*/, 'emphasis'],
+
+			//inline-literals
+			[/(``)((?:[^`]|\`(?!`))+)(``)/, ['', 'keyword', '']],
+			[/(__\s+)(.+)/, ['', 'keyword']],
+
+			//interpreted-text
+			[/(:)((?:@simpleRefNameWithoutBq)?)(:`)([^`]+)(`)/, ['', 'keyword', '', '', '']],
+			[/(`)([^`]+)(`:)((?:@simpleRefNameWithoutBq)?)(:)/, ['', '', '', 'keyword', '']],
+			[/(`)([^`]+)(`)/, ''],
+
+			//inline-internal-targets
+			[/(_`)(@phrase)(`)/, ['', 'string.link', '']],
+		],
+		citations: [
+			[/^(\.\.\s+\[)((?:@citationName))(\]\s+)(.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '', '']],
+		],
+		citationsReference: [
+			[/(\[)(@citationName)(\]_)/, ['', 'string.link', '']],
+		],
+		footnotes: [
+			[/^(\.\.\s+\[)((?:[0-9]+))(\]\s+.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '']],
+			[/^(\.\.\s+\[)((?:#@simpleRefName?))(\]\s+)(.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '', '']],
+			[/^(\.\.\s+\[)((?:\*))(\]\s+)(.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '', '']],
+		],
+		footnotesReference: [
+			[/(\[)([0-9]+)(\])(_)/, ['', 'string.link', '', '']],
+			[/(\[)(#@simpleRefName?)(\])(_)/, ['', 'string.link', '', '']],
+			[/(\[)(\*)(\])(_)/, ['', 'string.link', '', '']]
+		],
+		blankLineOfLiteralBlocks: [
+			[/^$/, '', '@subsequentLinesOfLiteralBlocks'],
+			[/^.*$/, '', '@pop'],
+		],
+		subsequentLinesOfLiteralBlocks: [
+			[/(@blockLiteralStart+)(.*)/, ['keyword', '']],
+			[/^(?!blockLiteralStart)/, '', '@popall']
+		],
+		subsequentLines: [
+			[/^[\s]+.*/, ''],
+			[/^(?!\s)/, '', '@pop'],
+		],
+		hyperlinks: [
+			[/^[\s]+.*/, 'string.link'],
+			[/^(?!\s)/, '', '@pop'],
+		],
+		comments: [
+			[/^[\s]+.*/, 'comment'],
+			[/^(?!\s)/, '', '@pop'],
+		],
+		tables: [
+			[/\+-[+-]+/, 'keyword'],
+			[/\+=[+=]+/, 'keyword'],
+		],
+	}
+};
+