123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171 |
- 'use strict';
- // token types (note: value shouldn't intersect with used char codes)
- var WHITESPACE = 1;
- var IDENTIFIER = 2;
- var NUMBER = 3;
- var STRING = 4;
- var COMMENT = 5;
- var PUNCTUATOR = 6;
- var CDO = 7;
- var CDC = 8;
- var ATRULE = 14;
- var FUNCTION = 15;
- var URL = 16;
- var RAW = 17;
- var TAB = 9;
- var N = 10;
- var F = 12;
- var R = 13;
- var SPACE = 32;
- var TYPE = {
- WhiteSpace: WHITESPACE,
- Identifier: IDENTIFIER,
- Number: NUMBER,
- String: STRING,
- Comment: COMMENT,
- Punctuator: PUNCTUATOR,
- CDO: CDO,
- CDC: CDC,
- Atrule: ATRULE,
- Function: FUNCTION,
- Url: URL,
- Raw: RAW,
- ExclamationMark: 33, // !
- QuotationMark: 34, // "
- NumberSign: 35, // #
- DollarSign: 36, // $
- PercentSign: 37, // %
- Ampersand: 38, // &
- Apostrophe: 39, // '
- LeftParenthesis: 40, // (
- RightParenthesis: 41, // )
- Asterisk: 42, // *
- PlusSign: 43, // +
- Comma: 44, // ,
- HyphenMinus: 45, // -
- FullStop: 46, // .
- Solidus: 47, // /
- Colon: 58, // :
- Semicolon: 59, // ;
- LessThanSign: 60, // <
- EqualsSign: 61, // =
- GreaterThanSign: 62, // >
- QuestionMark: 63, // ?
- CommercialAt: 64, // @
- LeftSquareBracket: 91, // [
- Backslash: 92, // \
- RightSquareBracket: 93, // ]
- CircumflexAccent: 94, // ^
- LowLine: 95, // _
- GraveAccent: 96, // `
- LeftCurlyBracket: 123, // {
- VerticalLine: 124, // |
- RightCurlyBracket: 125, // }
- Tilde: 126 // ~
- };
- var NAME = Object.keys(TYPE).reduce(function(result, key) {
- result[TYPE[key]] = key;
- return result;
- }, {});
- // https://drafts.csswg.org/css-syntax/#tokenizer-definitions
- // > non-ASCII code point
- // > A code point with a value equal to or greater than U+0080 <control>
- // > name-start code point
- // > A letter, a non-ASCII code point, or U+005F LOW LINE (_).
- // > name code point
- // > A name-start code point, a digit, or U+002D HYPHEN-MINUS (-)
- // That means only ASCII code points has a special meaning and we a maps for 0..127 codes only
- var SafeUint32Array = typeof Uint32Array !== 'undefined' ? Uint32Array : Array; // fallback on Array when TypedArray is not supported
- var SYMBOL_TYPE = new SafeUint32Array(0x80);
- var PUNCTUATION = new SafeUint32Array(0x80);
- var STOP_URL_RAW = new SafeUint32Array(0x80);
- for (var i = 0; i < SYMBOL_TYPE.length; i++) {
- SYMBOL_TYPE[i] = IDENTIFIER;
- }
- // fill categories
- [
- TYPE.ExclamationMark, // !
- TYPE.QuotationMark, // "
- TYPE.NumberSign, // #
- TYPE.DollarSign, // $
- TYPE.PercentSign, // %
- TYPE.Ampersand, // &
- TYPE.Apostrophe, // '
- TYPE.LeftParenthesis, // (
- TYPE.RightParenthesis, // )
- TYPE.Asterisk, // *
- TYPE.PlusSign, // +
- TYPE.Comma, // ,
- TYPE.HyphenMinus, // -
- TYPE.FullStop, // .
- TYPE.Solidus, // /
- TYPE.Colon, // :
- TYPE.Semicolon, // ;
- TYPE.LessThanSign, // <
- TYPE.EqualsSign, // =
- TYPE.GreaterThanSign, // >
- TYPE.QuestionMark, // ?
- TYPE.CommercialAt, // @
- TYPE.LeftSquareBracket, // [
- // TYPE.Backslash, // \
- TYPE.RightSquareBracket, // ]
- TYPE.CircumflexAccent, // ^
- // TYPE.LowLine, // _
- TYPE.GraveAccent, // `
- TYPE.LeftCurlyBracket, // {
- TYPE.VerticalLine, // |
- TYPE.RightCurlyBracket, // }
- TYPE.Tilde // ~
- ].forEach(function(key) {
- SYMBOL_TYPE[Number(key)] = PUNCTUATOR;
- PUNCTUATION[Number(key)] = PUNCTUATOR;
- });
- for (var i = 48; i <= 57; i++) {
- SYMBOL_TYPE[i] = NUMBER;
- }
- SYMBOL_TYPE[SPACE] = WHITESPACE;
- SYMBOL_TYPE[TAB] = WHITESPACE;
- SYMBOL_TYPE[N] = WHITESPACE;
- SYMBOL_TYPE[R] = WHITESPACE;
- SYMBOL_TYPE[F] = WHITESPACE;
- SYMBOL_TYPE[TYPE.Apostrophe] = STRING;
- SYMBOL_TYPE[TYPE.QuotationMark] = STRING;
- STOP_URL_RAW[SPACE] = 1;
- STOP_URL_RAW[TAB] = 1;
- STOP_URL_RAW[N] = 1;
- STOP_URL_RAW[R] = 1;
- STOP_URL_RAW[F] = 1;
- STOP_URL_RAW[TYPE.Apostrophe] = 1;
- STOP_URL_RAW[TYPE.QuotationMark] = 1;
- STOP_URL_RAW[TYPE.LeftParenthesis] = 1;
- STOP_URL_RAW[TYPE.RightParenthesis] = 1;
- // whitespace is punctuation ...
- PUNCTUATION[SPACE] = PUNCTUATOR;
- PUNCTUATION[TAB] = PUNCTUATOR;
- PUNCTUATION[N] = PUNCTUATOR;
- PUNCTUATION[R] = PUNCTUATOR;
- PUNCTUATION[F] = PUNCTUATOR;
- // ... hyper minus is not
- PUNCTUATION[TYPE.HyphenMinus] = 0;
- module.exports = {
- TYPE: TYPE,
- NAME: NAME,
- SYMBOL_TYPE: SYMBOL_TYPE,
- PUNCTUATION: PUNCTUATION,
- STOP_URL_RAW: STOP_URL_RAW
- };
|