splitIntoPotentialTokens.js 739 B

1234567891011121314151617181920212223242526272829303132333435363738394041
  1. // \n = 10
  2. // ; = 59
  3. // { = 123
  4. // } = 125
  5. // <space> = 32
  6. // \r = 13
  7. // \t = 9
  8. const splitIntoPotentialTokens = str => {
  9. const len = str.length;
  10. if (len === 0) return null;
  11. const results = [];
  12. let i = 0;
  13. for (; i < len; ) {
  14. const s = i;
  15. block: {
  16. let cc = str.charCodeAt(i);
  17. while (cc !== 10 && cc !== 59 && cc !== 123 && cc !== 125) {
  18. if (++i >= len) break block;
  19. cc = str.charCodeAt(i);
  20. }
  21. while (
  22. cc === 59 ||
  23. cc === 32 ||
  24. cc === 123 ||
  25. cc === 125 ||
  26. cc === 13 ||
  27. cc === 9
  28. ) {
  29. if (++i >= len) break block;
  30. cc = str.charCodeAt(i);
  31. }
  32. if (cc === 10) {
  33. i++;
  34. }
  35. }
  36. results.push(str.slice(s, i));
  37. }
  38. return results;
  39. };
  40. module.exports = splitIntoPotentialTokens;