tokenization.html 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325
  1. <!DOCTYPE html>
  2. <html>
  3. <head>
  4. <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
  5. <link rel="stylesheet" href="https://code.jquery.com/qunit/qunit-2.0.1.css">
  6. </head>
  7. <body>
  8. <script src="../metadata.js"></script>
  9. <script src="dev-setup.js"></script>
  10. <script src="tokenization.js"></script>
  11. <script src="https://code.jquery.com/qunit/qunit-2.0.1.js"></script>
  12. <script>
  13. var GENERATE_TOKENS = false;
  14. var LIMIT_TO_TEST = null;//'sample - xml';
  15. loadEditor(function() {
  16. let lazyModules = [
  17. 'vs/basic-languages/src/bat',
  18. 'vs/basic-languages/src/coffee',
  19. 'vs/basic-languages/src/cpp',
  20. 'vs/basic-languages/src/csharp',
  21. 'vs/basic-languages/src/dockerfile',
  22. 'vs/basic-languages/src/fsharp',
  23. 'vs/basic-languages/src/go',
  24. 'vs/basic-languages/src/handlebars',
  25. 'vs/basic-languages/src/html',
  26. 'vs/basic-languages/src/ini',
  27. 'vs/basic-languages/src/pug',
  28. 'vs/basic-languages/src/java',
  29. 'vs/basic-languages/src/lua',
  30. 'vs/basic-languages/src/markdown',
  31. 'vs/basic-languages/src/objective-c',
  32. 'vs/basic-languages/src/postiats',
  33. 'vs/basic-languages/src/php',
  34. 'vs/basic-languages/src/powershell',
  35. 'vs/basic-languages/src/python',
  36. 'vs/basic-languages/src/r',
  37. 'vs/basic-languages/src/razor',
  38. 'vs/basic-languages/src/ruby',
  39. 'vs/basic-languages/src/swift',
  40. 'vs/basic-languages/src/sql',
  41. 'vs/basic-languages/src/vb',
  42. 'vs/basic-languages/src/xml',
  43. 'vs/basic-languages/src/less',
  44. 'vs/basic-languages/src/scss',
  45. 'vs/basic-languages/src/css',
  46. 'vs/basic-languages/src/yaml',
  47. 'vs/language/typescript/src/mode',
  48. 'vs/language/css/cssMode',
  49. 'vs/language/json/jsonMode',
  50. 'vs/language/html/htmlMode'
  51. ];
  52. console.log('editor loaded');
  53. require(lazyModules, function() {
  54. console.log('all lazy code loaded');
  55. function resolveSample(sample) {
  56. return sample.loadText().then(function(txt) {
  57. return {
  58. name: sample.name,
  59. language: sample.mimeType,
  60. text: txt
  61. };
  62. });
  63. }
  64. require(['./samples'], function(SAMPLES) {
  65. Promise.all(
  66. SAMPLES.map(resolveSample)
  67. ).then(function(samples) {
  68. renderSamples(samples);
  69. });
  70. });
  71. });
  72. });
  73. function renderSamples(samples) {
  74. samples = samples.filter(function(sample) {
  75. if (typeof sample.language !== 'string') {
  76. return false;
  77. }
  78. if (sample.language === 'plaintext' || sample.language === 'text/plain') {
  79. return false;
  80. }
  81. // return sample.name === 'sample - css';
  82. return /sample/.test(sample.name);
  83. });
  84. samples.sort(function(a,b) {
  85. if (a.language === b.language) {
  86. if (a.text < b.text) {
  87. return -1;
  88. }
  89. if (a.text > b.text) {
  90. return 1;
  91. }
  92. return 0;
  93. }
  94. if (a.language < b.language) {
  95. return -1;
  96. }
  97. return 1;
  98. });
  99. samples.forEach(renderSample);
  100. generateTests();
  101. }
  102. var TEST_CASES = [];
  103. function renderSample(sample, index) {
  104. console.log('TEST #' + index + ': ' + sample.name);
  105. if (LIMIT_TO_TEST && sample.name !== LIMIT_TO_TEST) {
  106. TEST_CASES.push({ name: sample.name });
  107. return;
  108. }
  109. var testCase = {
  110. name: sample.name,
  111. // text: sample.text,
  112. language: sample.language,
  113. result: {
  114. vs: extractColors(sample, 'vs'),
  115. vs_dark: extractColors(sample, 'vs-dark'),
  116. hc_black: extractColors(sample, 'hc-black'),
  117. }
  118. };
  119. TEST_CASES.push(testCase);
  120. }
  121. function extractColors(sample, theme) {
  122. var out = document.createElement('pre');
  123. var txt = document.createTextNode(sample.text);
  124. out.appendChild(txt);
  125. document.body.appendChild(out);
  126. monaco.editor.colorizeElement(out, {
  127. theme: theme,
  128. mimeType: sample.language
  129. });
  130. if (GENERATE_TOKENS) {
  131. var tokens = monaco.editor.tokenize(sample.text || '', sample.language);
  132. }
  133. // console.log(out);
  134. var lineElement = out.firstElementChild;
  135. var allResult = [];
  136. while (lineElement) {
  137. // console.log(lineElement);
  138. var pieces = lineElement.children;
  139. var result = [];
  140. if (GENERATE_TOKENS) {
  141. var lineTokens = tokens.shift();
  142. if (lineTokens.length > 0) {
  143. for (var i = 0; i < lineTokens.length - 1; i++) {
  144. lineTokens[i].length = lineTokens[i + 1].offset - lineTokens[i].offset;
  145. }
  146. lineTokens[lineTokens.length - 1].length = 1000000;
  147. }
  148. }
  149. for (var i = 0; i < pieces.length; i++) {
  150. var piece = pieces[i];
  151. // console.log(piece);
  152. var text = piece.innerText;
  153. var r = window.getComputedStyle(piece);
  154. var color = rgbToHex(r.color);
  155. // var isWhitespace = /^\s+$/.test(text);
  156. // var partTokens = undefined;
  157. if (GENERATE_TOKENS) {
  158. let partTokens = [];
  159. let partText = text;
  160. while (lineTokens.length > 0 && partText.length > 0) {
  161. let t = lineTokens.shift();
  162. partTokens.push(t);
  163. partText = partText.substr(t.length);
  164. }
  165. result.push({
  166. text: text,
  167. color: color,
  168. tokens: partTokens
  169. });
  170. } else {
  171. result.push({
  172. text: text,
  173. color: color,
  174. });
  175. }
  176. }
  177. allResult.push(result);
  178. lineElement = lineElement.nextElementSibling.nextElementSibling;
  179. }
  180. // console.log(JSON.stringify(allResult));
  181. out.parentNode.removeChild(out);
  182. return allResult;
  183. }
  184. var rgbCache = {};
  185. function rgbToHex(rgb) {
  186. if (!rgbCache[rgb]) {
  187. var m = rgb.match(/rgb\((\d+), (\d+), (\d+)\)/);
  188. var r = parseInt(m[1], 10);
  189. var g = parseInt(m[2], 10);
  190. var b = parseInt(m[3], 10);
  191. var r = '#' + to2Hex(r) + to2Hex(g) + to2Hex(b);
  192. rgbCache[rgb] = r;
  193. }
  194. return rgbCache[rgb];
  195. }
  196. function to2Hex(n) {
  197. var r = n.toString(16);
  198. if (r.length === 1) {
  199. return '0' + r;
  200. }
  201. return r;
  202. }
  203. function generateTests() {
  204. // var ta = document.createElement('textarea');
  205. // ta.value = JSON.stringify(TEST_CASES, null, '\t');
  206. // ta.style.display = 'fixed';
  207. // ta.style.top = '0';
  208. // ta.style.left = '0';
  209. // document.body.appendChild(ta);
  210. // ta.select();
  211. // return;
  212. for (var i = 0, len = TEST_CASES.length; i < len; i++) {
  213. if (LIMIT_TO_TEST && TEST_CASES[i].name !== LIMIT_TO_TEST) {
  214. // if (TEST_CASES[i].name !== 'sample - html') {
  215. continue;
  216. }
  217. generateTest(TEST_CASES[i], EXPECTED[i]);
  218. }
  219. QUnit.start();
  220. }
  221. function mergeEqualColors(source) {
  222. // return source;
  223. let result = [];
  224. for (let i = 0; i < source.length; i++) {
  225. let color = source[i].color;
  226. let text = source[i].text;
  227. if (result.length > 0 && result[result.length - 1].color === color) {
  228. result[result.length - 1].text += text;
  229. } else {
  230. result.push({
  231. color: color,
  232. text: text
  233. });
  234. }
  235. }
  236. return result;
  237. }
  238. function generateTest(actual, expected) {
  239. ['vs', 'vs_dark', 'hc_black'].forEach(function(theme) {
  240. // if (actual.name !== 'sample - bat') {
  241. // return;
  242. // }
  243. QUnit.test(actual.name + ' ' + theme, function(assert) {
  244. var _actual = actual.result[theme];
  245. var _expected = expected.result[theme];
  246. for (var i = 0; i < _actual.length; i++) {
  247. var lineActual = mergeEqualColors(_actual[i]);
  248. // .slice(0);
  249. // for (var j = 1; j < lineActual.length; j++) {
  250. // if (lineActual[j-1].color === lineActual[j].color) {
  251. // lineActual[j-1] = {
  252. // color: lineActual[j-1].color,
  253. // text: lineActual[j-1].text + lineActual[j].text
  254. // };
  255. // lineActual.splice(j, 1);
  256. // j--;
  257. // }
  258. // }
  259. var lineExpected = mergeEqualColors(_expected[i]);
  260. // .slice(0);
  261. // for (var j = 1; j < lineExpected.length; j++) {
  262. // if (lineExpected[j-1].color === lineExpected[j].color) {
  263. // lineExpected[j-1] = {
  264. // color: lineExpected[j-1].color,
  265. // text: lineExpected[j-1].text + lineExpected[j].text
  266. // };
  267. // // lineExpected[j-1].text += lineExpected[j].text;
  268. // lineExpected.splice(j, 1);
  269. // j--;
  270. // }
  271. // }
  272. assert.deepEqual(lineActual, lineExpected, 'Line #' + (i+1));
  273. }
  274. });
  275. });
  276. }
  277. QUnit.config.autostart = false;
  278. </script>
  279. <div id="qunit"></div>
  280. <div id="qunit-fixture"></div>
  281. </body>
  282. </html>