remove-duplicates.js 978 B

12345678910111213141516171819202122232425262728293031323334353637383940414243
  1. var Token = require('../../tokenizer/token');
  2. var serializeBody = require('../../writer/one-time').body;
  3. var serializeRules = require('../../writer/one-time').rules;
  4. function removeDuplicates(tokens) {
  5. var matched = {};
  6. var moreThanOnce = [];
  7. var id, token;
  8. var body, bodies;
  9. for (var i = 0, l = tokens.length; i < l; i++) {
  10. token = tokens[i];
  11. if (token[0] != Token.RULE)
  12. continue;
  13. id = serializeRules(token[1]);
  14. if (matched[id] && matched[id].length == 1)
  15. moreThanOnce.push(id);
  16. else
  17. matched[id] = matched[id] || [];
  18. matched[id].push(i);
  19. }
  20. for (i = 0, l = moreThanOnce.length; i < l; i++) {
  21. id = moreThanOnce[i];
  22. bodies = [];
  23. for (var j = matched[id].length - 1; j >= 0; j--) {
  24. token = tokens[matched[id][j]];
  25. body = serializeBody(token[2]);
  26. if (bodies.indexOf(body) > -1)
  27. token[2] = [];
  28. else
  29. bodies.push(body);
  30. }
  31. }
  32. }
  33. module.exports = removeDuplicates;