const.js 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171
  1. 'use strict';
  2. // token types (note: value shouldn't intersect with used char codes)
  3. var WHITESPACE = 1;
  4. var IDENTIFIER = 2;
  5. var NUMBER = 3;
  6. var STRING = 4;
  7. var COMMENT = 5;
  8. var PUNCTUATOR = 6;
  9. var CDO = 7;
  10. var CDC = 8;
  11. var ATRULE = 14;
  12. var FUNCTION = 15;
  13. var URL = 16;
  14. var RAW = 17;
  15. var TAB = 9;
  16. var N = 10;
  17. var F = 12;
  18. var R = 13;
  19. var SPACE = 32;
  20. var TYPE = {
  21. WhiteSpace: WHITESPACE,
  22. Identifier: IDENTIFIER,
  23. Number: NUMBER,
  24. String: STRING,
  25. Comment: COMMENT,
  26. Punctuator: PUNCTUATOR,
  27. CDO: CDO,
  28. CDC: CDC,
  29. Atrule: ATRULE,
  30. Function: FUNCTION,
  31. Url: URL,
  32. Raw: RAW,
  33. ExclamationMark: 33, // !
  34. QuotationMark: 34, // "
  35. NumberSign: 35, // #
  36. DollarSign: 36, // $
  37. PercentSign: 37, // %
  38. Ampersand: 38, // &
  39. Apostrophe: 39, // '
  40. LeftParenthesis: 40, // (
  41. RightParenthesis: 41, // )
  42. Asterisk: 42, // *
  43. PlusSign: 43, // +
  44. Comma: 44, // ,
  45. HyphenMinus: 45, // -
  46. FullStop: 46, // .
  47. Solidus: 47, // /
  48. Colon: 58, // :
  49. Semicolon: 59, // ;
  50. LessThanSign: 60, // <
  51. EqualsSign: 61, // =
  52. GreaterThanSign: 62, // >
  53. QuestionMark: 63, // ?
  54. CommercialAt: 64, // @
  55. LeftSquareBracket: 91, // [
  56. Backslash: 92, // \
  57. RightSquareBracket: 93, // ]
  58. CircumflexAccent: 94, // ^
  59. LowLine: 95, // _
  60. GraveAccent: 96, // `
  61. LeftCurlyBracket: 123, // {
  62. VerticalLine: 124, // |
  63. RightCurlyBracket: 125, // }
  64. Tilde: 126 // ~
  65. };
  66. var NAME = Object.keys(TYPE).reduce(function(result, key) {
  67. result[TYPE[key]] = key;
  68. return result;
  69. }, {});
  70. // https://drafts.csswg.org/css-syntax/#tokenizer-definitions
  71. // > non-ASCII code point
  72. // > A code point with a value equal to or greater than U+0080 <control>
  73. // > name-start code point
  74. // > A letter, a non-ASCII code point, or U+005F LOW LINE (_).
  75. // > name code point
  76. // > A name-start code point, a digit, or U+002D HYPHEN-MINUS (-)
  77. // That means only ASCII code points has a special meaning and we a maps for 0..127 codes only
  78. var SafeUint32Array = typeof Uint32Array !== 'undefined' ? Uint32Array : Array; // fallback on Array when TypedArray is not supported
  79. var SYMBOL_TYPE = new SafeUint32Array(0x80);
  80. var PUNCTUATION = new SafeUint32Array(0x80);
  81. var STOP_URL_RAW = new SafeUint32Array(0x80);
  82. for (var i = 0; i < SYMBOL_TYPE.length; i++) {
  83. SYMBOL_TYPE[i] = IDENTIFIER;
  84. }
  85. // fill categories
  86. [
  87. TYPE.ExclamationMark, // !
  88. TYPE.QuotationMark, // "
  89. TYPE.NumberSign, // #
  90. TYPE.DollarSign, // $
  91. TYPE.PercentSign, // %
  92. TYPE.Ampersand, // &
  93. TYPE.Apostrophe, // '
  94. TYPE.LeftParenthesis, // (
  95. TYPE.RightParenthesis, // )
  96. TYPE.Asterisk, // *
  97. TYPE.PlusSign, // +
  98. TYPE.Comma, // ,
  99. TYPE.HyphenMinus, // -
  100. TYPE.FullStop, // .
  101. TYPE.Solidus, // /
  102. TYPE.Colon, // :
  103. TYPE.Semicolon, // ;
  104. TYPE.LessThanSign, // <
  105. TYPE.EqualsSign, // =
  106. TYPE.GreaterThanSign, // >
  107. TYPE.QuestionMark, // ?
  108. TYPE.CommercialAt, // @
  109. TYPE.LeftSquareBracket, // [
  110. // TYPE.Backslash, // \
  111. TYPE.RightSquareBracket, // ]
  112. TYPE.CircumflexAccent, // ^
  113. // TYPE.LowLine, // _
  114. TYPE.GraveAccent, // `
  115. TYPE.LeftCurlyBracket, // {
  116. TYPE.VerticalLine, // |
  117. TYPE.RightCurlyBracket, // }
  118. TYPE.Tilde // ~
  119. ].forEach(function(key) {
  120. SYMBOL_TYPE[Number(key)] = PUNCTUATOR;
  121. PUNCTUATION[Number(key)] = PUNCTUATOR;
  122. });
  123. for (var i = 48; i <= 57; i++) {
  124. SYMBOL_TYPE[i] = NUMBER;
  125. }
  126. SYMBOL_TYPE[SPACE] = WHITESPACE;
  127. SYMBOL_TYPE[TAB] = WHITESPACE;
  128. SYMBOL_TYPE[N] = WHITESPACE;
  129. SYMBOL_TYPE[R] = WHITESPACE;
  130. SYMBOL_TYPE[F] = WHITESPACE;
  131. SYMBOL_TYPE[TYPE.Apostrophe] = STRING;
  132. SYMBOL_TYPE[TYPE.QuotationMark] = STRING;
  133. STOP_URL_RAW[SPACE] = 1;
  134. STOP_URL_RAW[TAB] = 1;
  135. STOP_URL_RAW[N] = 1;
  136. STOP_URL_RAW[R] = 1;
  137. STOP_URL_RAW[F] = 1;
  138. STOP_URL_RAW[TYPE.Apostrophe] = 1;
  139. STOP_URL_RAW[TYPE.QuotationMark] = 1;
  140. STOP_URL_RAW[TYPE.LeftParenthesis] = 1;
  141. STOP_URL_RAW[TYPE.RightParenthesis] = 1;
  142. // whitespace is punctuation ...
  143. PUNCTUATION[SPACE] = PUNCTUATOR;
  144. PUNCTUATION[TAB] = PUNCTUATOR;
  145. PUNCTUATION[N] = PUNCTUATOR;
  146. PUNCTUATION[R] = PUNCTUATOR;
  147. PUNCTUATION[F] = PUNCTUATOR;
  148. // ... hyper minus is not
  149. PUNCTUATION[TYPE.HyphenMinus] = 0;
  150. module.exports = {
  151. TYPE: TYPE,
  152. NAME: NAME,
  153. SYMBOL_TYPE: SYMBOL_TYPE,
  154. PUNCTUATION: PUNCTUATION,
  155. STOP_URL_RAW: STOP_URL_RAW
  156. };