apex.test.ts 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651
  1. /*---------------------------------------------------------------------------------------------
  2. * Copyright (c) Microsoft Corporation. All rights reserved.
  3. * Licensed under the MIT License. See License.txt in the project root for license information.
  4. *--------------------------------------------------------------------------------------------*/
  5. 'use strict';
  6. import { testTokenization } from '../test/testRunner';
  7. testTokenization('apex', [
  8. // Comments - single line
  9. [{
  10. line: '//',
  11. tokens: [
  12. { startIndex: 0, type: 'comment.apex' }
  13. ]
  14. }],
  15. [{
  16. line: ' // a comment',
  17. tokens: [
  18. { startIndex: 0, type: '' },
  19. { startIndex: 4, type: 'comment.apex' }
  20. ]
  21. }],
  22. // Broken nested tokens due to invalid comment tokenization
  23. [{
  24. line: '/* //*/ a',
  25. tokens: [
  26. { startIndex: 0, type: 'comment.apex' },
  27. { startIndex: 7, type: '' },
  28. { startIndex: 8, type: 'identifier.apex' }
  29. ]
  30. }],
  31. [{
  32. line: '// a comment',
  33. tokens: [
  34. { startIndex: 0, type: 'comment.apex' }
  35. ]
  36. }],
  37. [{
  38. line: '//sticky comment',
  39. tokens: [
  40. { startIndex: 0, type: 'comment.apex' }
  41. ]
  42. }],
  43. [{
  44. line: '/almost a comment',
  45. tokens: [
  46. { startIndex: 0, type: 'delimiter.apex' },
  47. { startIndex: 1, type: 'identifier.apex' },
  48. { startIndex: 7, type: '' },
  49. { startIndex: 8, type: 'identifier.apex' },
  50. { startIndex: 9, type: '' },
  51. { startIndex: 10, type: 'identifier.apex' }
  52. ]
  53. }],
  54. [{
  55. line: '1 / 2; /* comment',
  56. tokens: [
  57. { startIndex: 0, type: 'number.apex' },
  58. { startIndex: 1, type: '' },
  59. { startIndex: 2, type: 'delimiter.apex' },
  60. { startIndex: 3, type: '' },
  61. { startIndex: 4, type: 'number.apex' },
  62. { startIndex: 5, type: 'delimiter.apex' },
  63. { startIndex: 6, type: '' },
  64. { startIndex: 7, type: 'comment.apex' }
  65. ]
  66. }],
  67. [{
  68. line: 'int x = 1; // my comment // is a nice one',
  69. tokens: [
  70. { startIndex: 0, type: 'keyword.int.apex' },
  71. { startIndex: 3, type: '' },
  72. { startIndex: 4, type: 'identifier.apex' },
  73. { startIndex: 5, type: '' },
  74. { startIndex: 6, type: 'delimiter.apex' },
  75. { startIndex: 7, type: '' },
  76. { startIndex: 8, type: 'number.apex' },
  77. { startIndex: 9, type: 'delimiter.apex' },
  78. { startIndex: 10, type: '' },
  79. { startIndex: 11, type: 'comment.apex' }
  80. ]
  81. }],
  82. // Comments - range comment, single line
  83. [{
  84. line: '/* a simple comment */',
  85. tokens: [
  86. { startIndex: 0, type: 'comment.apex' }
  87. ]
  88. }],
  89. [{
  90. line: 'int x = /* a simple comment */ 1;',
  91. tokens: [
  92. { startIndex: 0, type: 'keyword.int.apex' },
  93. { startIndex: 3, type: '' },
  94. { startIndex: 4, type: 'identifier.apex' },
  95. { startIndex: 5, type: '' },
  96. { startIndex: 6, type: 'delimiter.apex' },
  97. { startIndex: 7, type: '' },
  98. { startIndex: 8, type: 'comment.apex' },
  99. { startIndex: 30, type: '' },
  100. { startIndex: 31, type: 'number.apex' },
  101. { startIndex: 32, type: 'delimiter.apex' }
  102. ]
  103. }],
  104. [{
  105. line: 'int x = /* comment */ 1; */',
  106. tokens: [
  107. { startIndex: 0, type: 'keyword.int.apex' },
  108. { startIndex: 3, type: '' },
  109. { startIndex: 4, type: 'identifier.apex' },
  110. { startIndex: 5, type: '' },
  111. { startIndex: 6, type: 'delimiter.apex' },
  112. { startIndex: 7, type: '' },
  113. { startIndex: 8, type: 'comment.apex' },
  114. { startIndex: 21, type: '' },
  115. { startIndex: 22, type: 'number.apex' },
  116. { startIndex: 23, type: 'delimiter.apex' },
  117. { startIndex: 24, type: '' }
  118. ]
  119. }],
  120. [{
  121. line: 'x = /**/;',
  122. tokens: [
  123. { startIndex: 0, type: 'identifier.apex' },
  124. { startIndex: 1, type: '' },
  125. { startIndex: 2, type: 'delimiter.apex' },
  126. { startIndex: 3, type: '' },
  127. { startIndex: 4, type: 'comment.apex' },
  128. { startIndex: 8, type: 'delimiter.apex' }
  129. ]
  130. }],
  131. [{
  132. line: 'x = /*/;',
  133. tokens: [
  134. { startIndex: 0, type: 'identifier.apex' },
  135. { startIndex: 1, type: '' },
  136. { startIndex: 2, type: 'delimiter.apex' },
  137. { startIndex: 3, type: '' },
  138. { startIndex: 4, type: 'comment.apex' }
  139. ]
  140. }],
  141. // Comments - range comment, multiple lines
  142. [{
  143. line: '/* start of multiline comment',
  144. tokens: [
  145. { startIndex: 0, type: 'comment.apex' }
  146. ]
  147. }, {
  148. line: 'a comment between without a star',
  149. tokens: [
  150. { startIndex: 0, type: 'comment.apex' }
  151. ]
  152. }, {
  153. line: 'end of multiline comment*/',
  154. tokens: [
  155. { startIndex: 0, type: 'comment.apex' }
  156. ]
  157. }],
  158. [{
  159. line: 'int x = /* start a comment',
  160. tokens: [
  161. { startIndex: 0, type: 'keyword.int.apex' },
  162. { startIndex: 3, type: '' },
  163. { startIndex: 4, type: 'identifier.apex' },
  164. { startIndex: 5, type: '' },
  165. { startIndex: 6, type: 'delimiter.apex' },
  166. { startIndex: 7, type: '' },
  167. { startIndex: 8, type: 'comment.apex' }
  168. ]
  169. }, {
  170. line: ' a ',
  171. tokens: [
  172. { startIndex: 0, type: 'comment.apex' }
  173. ]
  174. }, {
  175. line: 'and end it */ 2;',
  176. tokens: [
  177. { startIndex: 0, type: 'comment.apex' },
  178. { startIndex: 13, type: '' },
  179. { startIndex: 14, type: 'number.apex' },
  180. { startIndex: 15, type: 'delimiter.apex' }
  181. ]
  182. }],
  183. // Comments - apex doc, multiple lines
  184. [{
  185. line: '/** start of Apex Doc',
  186. tokens: [
  187. { startIndex: 0, type: 'comment.doc.apex' }
  188. ]
  189. }, {
  190. line: 'a comment between without a star',
  191. tokens: [
  192. { startIndex: 0, type: 'comment.doc.apex' }
  193. ]
  194. }, {
  195. line: 'end of multiline comment*/',
  196. tokens: [
  197. { startIndex: 0, type: 'comment.doc.apex' }
  198. ]
  199. }],
  200. // Keywords
  201. [{
  202. line: 'package test; class Program { static void main(String[] args) {} } }',
  203. tokens: [
  204. { startIndex: 0, type: 'keyword.package.apex' },
  205. { startIndex: 7, type: '' },
  206. { startIndex: 8, type: 'identifier.apex' },
  207. { startIndex: 12, type: 'delimiter.apex' },
  208. { startIndex: 13, type: '' },
  209. { startIndex: 14, type: 'keyword.class.apex' },
  210. { startIndex: 19, type: '' },
  211. { startIndex: 20, type: 'type.identifier.apex' },
  212. { startIndex: 27, type: '' },
  213. { startIndex: 28, type: 'delimiter.curly.apex' },
  214. { startIndex: 29, type: '' },
  215. { startIndex: 30, type: 'keyword.static.apex' },
  216. { startIndex: 36, type: '' },
  217. { startIndex: 37, type: 'keyword.void.apex' },
  218. { startIndex: 41, type: '' },
  219. { startIndex: 42, type: 'identifier.apex' },
  220. { startIndex: 46, type: 'delimiter.parenthesis.apex' },
  221. { startIndex: 47, type: 'type.identifier.apex' },
  222. { startIndex: 53, type: 'delimiter.square.apex' },
  223. { startIndex: 55, type: '' },
  224. { startIndex: 56, type: 'identifier.apex' },
  225. { startIndex: 60, type: 'delimiter.parenthesis.apex' },
  226. { startIndex: 61, type: '' },
  227. { startIndex: 62, type: 'delimiter.curly.apex' },
  228. { startIndex: 64, type: '' },
  229. { startIndex: 65, type: 'delimiter.curly.apex' },
  230. { startIndex: 66, type: '' },
  231. { startIndex: 67, type: 'delimiter.curly.apex' }
  232. ]
  233. }],
  234. // Keywords with case variations
  235. [{
  236. line: 'Package test; CLASS Program { Static void main(String[] args) {} } }',
  237. tokens: [
  238. { startIndex: 0, type: 'keyword.Package.apex' },
  239. { startIndex: 7, type: '' },
  240. { startIndex: 8, type: 'identifier.apex' },
  241. { startIndex: 12, type: 'delimiter.apex' },
  242. { startIndex: 13, type: '' },
  243. { startIndex: 14, type: 'keyword.CLASS.apex' },
  244. { startIndex: 19, type: '' },
  245. { startIndex: 20, type: 'type.identifier.apex' },
  246. { startIndex: 27, type: '' },
  247. { startIndex: 28, type: 'delimiter.curly.apex' },
  248. { startIndex: 29, type: '' },
  249. { startIndex: 30, type: 'keyword.Static.apex' },
  250. { startIndex: 36, type: '' },
  251. { startIndex: 37, type: 'keyword.void.apex' },
  252. { startIndex: 41, type: '' },
  253. { startIndex: 42, type: 'identifier.apex' },
  254. { startIndex: 46, type: 'delimiter.parenthesis.apex' },
  255. { startIndex: 47, type: 'type.identifier.apex' },
  256. { startIndex: 53, type: 'delimiter.square.apex' },
  257. { startIndex: 55, type: '' },
  258. { startIndex: 56, type: 'identifier.apex' },
  259. { startIndex: 60, type: 'delimiter.parenthesis.apex' },
  260. { startIndex: 61, type: '' },
  261. { startIndex: 62, type: 'delimiter.curly.apex' },
  262. { startIndex: 64, type: '' },
  263. { startIndex: 65, type: 'delimiter.curly.apex' },
  264. { startIndex: 66, type: '' },
  265. { startIndex: 67, type: 'delimiter.curly.apex' }
  266. ]
  267. }],
  268. // Numbers
  269. [{
  270. line: '0',
  271. tokens: [
  272. { startIndex: 0, type: 'number.apex' }
  273. ]
  274. }],
  275. [{
  276. line: '0.10',
  277. tokens: [
  278. { startIndex: 0, type: 'number.float.apex' }
  279. ]
  280. }],
  281. [{
  282. line: '0x',
  283. tokens: [
  284. { startIndex: 0, type: 'number.apex' },
  285. { startIndex: 1, type: 'identifier.apex' }
  286. ]
  287. }],
  288. [{
  289. line: '10e3',
  290. tokens: [
  291. { startIndex: 0, type: 'number.float.apex' }
  292. ]
  293. }],
  294. [{
  295. line: '10f',
  296. tokens: [
  297. { startIndex: 0, type: 'number.float.apex' }
  298. ]
  299. }],
  300. [{
  301. line: '23.5',
  302. tokens: [
  303. { startIndex: 0, type: 'number.float.apex' }
  304. ]
  305. }],
  306. [{
  307. line: '23.5e3',
  308. tokens: [
  309. { startIndex: 0, type: 'number.float.apex' }
  310. ]
  311. }],
  312. [{
  313. line: '23.5e-3',
  314. tokens: [
  315. { startIndex: 0, type: 'number.float.apex' }
  316. ]
  317. }],
  318. [{
  319. line: '23.5E3',
  320. tokens: [
  321. { startIndex: 0, type: 'number.float.apex' }
  322. ]
  323. }],
  324. [{
  325. line: '23.5E-3',
  326. tokens: [
  327. { startIndex: 0, type: 'number.float.apex' }
  328. ]
  329. }],
  330. [{
  331. line: '23.5F',
  332. tokens: [
  333. { startIndex: 0, type: 'number.float.apex' }
  334. ]
  335. }],
  336. [{
  337. line: '23.5f',
  338. tokens: [
  339. { startIndex: 0, type: 'number.float.apex' }
  340. ]
  341. }],
  342. [{
  343. line: '23.5D',
  344. tokens: [
  345. { startIndex: 0, type: 'number.float.apex' }
  346. ]
  347. }],
  348. [{
  349. line: '23.5d',
  350. tokens: [
  351. { startIndex: 0, type: 'number.float.apex' }
  352. ]
  353. }],
  354. [{
  355. line: '1.72E3D',
  356. tokens: [
  357. { startIndex: 0, type: 'number.float.apex' }
  358. ]
  359. }],
  360. [{
  361. line: '1.72E3d',
  362. tokens: [
  363. { startIndex: 0, type: 'number.float.apex' }
  364. ]
  365. }],
  366. [{
  367. line: '1.72E-3d',
  368. tokens: [
  369. { startIndex: 0, type: 'number.float.apex' }
  370. ]
  371. }],
  372. [{
  373. line: '1.72e3D',
  374. tokens: [
  375. { startIndex: 0, type: 'number.float.apex' }
  376. ]
  377. }],
  378. [{
  379. line: '1.72e3d',
  380. tokens: [
  381. { startIndex: 0, type: 'number.float.apex' }
  382. ]
  383. }],
  384. [{
  385. line: '1.72e-3d',
  386. tokens: [
  387. { startIndex: 0, type: 'number.float.apex' }
  388. ]
  389. }],
  390. [{
  391. line: '23L',
  392. tokens: [
  393. { startIndex: 0, type: 'number.apex' }
  394. ]
  395. }],
  396. [{
  397. line: '23l',
  398. tokens: [
  399. { startIndex: 0, type: 'number.apex' }
  400. ]
  401. }],
  402. [{
  403. line: '0_52',
  404. tokens: [
  405. { startIndex: 0, type: 'number.apex' }
  406. ]
  407. }],
  408. [{
  409. line: '5_2',
  410. tokens: [
  411. { startIndex: 0, type: 'number.apex' }
  412. ]
  413. }],
  414. [{
  415. line: '5_______2',
  416. tokens: [
  417. { startIndex: 0, type: 'number.apex' }
  418. ]
  419. }],
  420. [{
  421. line: '3_.1415F',
  422. tokens: [
  423. { startIndex: 0, type: 'number.apex' },
  424. { startIndex: 1, type: 'identifier.apex' },
  425. { startIndex: 2, type: 'delimiter.apex' },
  426. { startIndex: 3, type: 'number.float.apex' }
  427. ]
  428. }],
  429. [{
  430. line: '3._1415F',
  431. tokens: [
  432. { startIndex: 0, type: 'number.apex' },
  433. { startIndex: 1, type: 'delimiter.apex' },
  434. { startIndex: 2, type: 'identifier.apex' }
  435. ]
  436. }],
  437. [{
  438. line: '999_99_9999_L',
  439. tokens: [
  440. { startIndex: 0, type: 'number.apex' },
  441. { startIndex: 11, type: 'identifier.apex' }
  442. ]
  443. }],
  444. [{
  445. line: '52_',
  446. tokens: [
  447. { startIndex: 0, type: 'number.apex' },
  448. { startIndex: 2, type: 'identifier.apex' }
  449. ]
  450. }],
  451. [{
  452. line: '0_x52',
  453. tokens: [
  454. { startIndex: 0, type: 'number.apex' },
  455. { startIndex: 1, type: 'identifier.apex' }
  456. ]
  457. }],
  458. [{
  459. line: '0x_52',
  460. tokens: [
  461. { startIndex: 0, type: 'number.apex' },
  462. { startIndex: 1, type: 'identifier.apex' }
  463. ]
  464. }],
  465. [{
  466. line: '23.5L',
  467. tokens: [
  468. { startIndex: 0, type: 'number.float.apex' },
  469. { startIndex: 4, type: 'type.identifier.apex' }
  470. ]
  471. }],
  472. [{
  473. line: '0+0',
  474. tokens: [
  475. { startIndex: 0, type: 'number.apex' },
  476. { startIndex: 1, type: 'delimiter.apex' },
  477. { startIndex: 2, type: 'number.apex' }
  478. ]
  479. }],
  480. [{
  481. line: '100+10',
  482. tokens: [
  483. { startIndex: 0, type: 'number.apex' },
  484. { startIndex: 3, type: 'delimiter.apex' },
  485. { startIndex: 4, type: 'number.apex' }
  486. ]
  487. }],
  488. [{
  489. line: '0 + 0',
  490. tokens: [
  491. { startIndex: 0, type: 'number.apex' },
  492. { startIndex: 1, type: '' },
  493. { startIndex: 2, type: 'delimiter.apex' },
  494. { startIndex: 3, type: '' },
  495. { startIndex: 4, type: 'number.apex' }
  496. ]
  497. }],
  498. // single line Strings
  499. [{
  500. line: 'String s = "I\'m an Apex String";',
  501. tokens: [
  502. { startIndex: 0, type: 'type.identifier.apex' },
  503. { startIndex: 6, type: '' },
  504. { startIndex: 7, type: 'identifier.apex' },
  505. { startIndex: 8, type: '' },
  506. { startIndex: 9, type: 'delimiter.apex' },
  507. { startIndex: 10, type: '' },
  508. { startIndex: 11, type: 'string.apex' },
  509. { startIndex: 31, type: 'delimiter.apex' }
  510. ]
  511. }],
  512. [{
  513. line: 'String s = "concatenated" + " String" ;',
  514. tokens: [
  515. { startIndex: 0, type: 'type.identifier.apex' },
  516. { startIndex: 6, type: '' },
  517. { startIndex: 7, type: 'identifier.apex' },
  518. { startIndex: 8, type: '' },
  519. { startIndex: 9, type: 'delimiter.apex' },
  520. { startIndex: 10, type: '' },
  521. { startIndex: 11, type: 'string.apex' },
  522. { startIndex: 25, type: '' },
  523. { startIndex: 26, type: 'delimiter.apex' },
  524. { startIndex: 27, type: '' },
  525. { startIndex: 28, type: 'string.apex' },
  526. { startIndex: 37, type: '' },
  527. { startIndex: 38, type: 'delimiter.apex' }
  528. ]
  529. }],
  530. [{
  531. line: '"quote in a string"',
  532. tokens: [
  533. { startIndex: 0, type: 'string.apex' }
  534. ]
  535. }],
  536. [{
  537. line: '"escaping \\"quotes\\" is cool"',
  538. tokens: [
  539. { startIndex: 0, type: 'string.apex' },
  540. { startIndex: 10, type: 'string.escape.apex' },
  541. { startIndex: 12, type: 'string.apex' },
  542. { startIndex: 18, type: 'string.escape.apex' },
  543. { startIndex: 20, type: 'string.apex' }
  544. ]
  545. }],
  546. [{
  547. line: '"\\"',
  548. tokens: [
  549. { startIndex: 0, type: 'string.invalid.apex' }
  550. ]
  551. }],
  552. // Annotations
  553. [{
  554. line: '@',
  555. tokens: [
  556. { startIndex: 0, type: '' }
  557. ]
  558. }],
  559. [{
  560. line: '@Override',
  561. tokens: [
  562. { startIndex: 0, type: 'annotation.apex' }
  563. ]
  564. }],
  565. [{
  566. line: '@SuppressWarnings(value = "aString")',
  567. tokens: [
  568. { startIndex: 0, type: 'annotation.apex' },
  569. { startIndex: 17, type: 'delimiter.parenthesis.apex' },
  570. { startIndex: 18, type: 'identifier.apex' },
  571. { startIndex: 23, type: '' },
  572. { startIndex: 24, type: 'delimiter.apex' },
  573. { startIndex: 25, type: '' },
  574. { startIndex: 26, type: 'string.apex' },
  575. { startIndex: 35, type: 'delimiter.parenthesis.apex' }
  576. ]
  577. }],
  578. [{
  579. line: '@ AnnotationWithKeywordAfter private',
  580. tokens: [
  581. { startIndex: 0, type: 'annotation.apex' },
  582. { startIndex: 28, type: '' },
  583. { startIndex: 29, type: 'keyword.private.apex' }
  584. ]
  585. }]
  586. ]);