utils.js
7.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
var charCodeDef = require('./char-code-definitions');
var isDigit = charCodeDef.isDigit;
var isHexDigit = charCodeDef.isHexDigit;
var isUppercaseLetter = charCodeDef.isUppercaseLetter;
var isName = charCodeDef.isName;
var isWhiteSpace = charCodeDef.isWhiteSpace;
var isValidEscape = charCodeDef.isValidEscape;
function getCharCode(source, offset) {
return offset < source.length ? source.charCodeAt(offset) : 0;
}
function getNewlineLength(source, offset, code) {
if (code === 13 /* \r */ && getCharCode(source, offset + 1) === 10 /* \n */) {
return 2;
}
return 1;
}
function cmpChar(testStr, offset, referenceCode) {
var code = testStr.charCodeAt(offset);
// code.toLowerCase() for A..Z
if (isUppercaseLetter(code)) {
code = code | 32;
}
return code === referenceCode;
}
function cmpStr(testStr, start, end, referenceStr) {
if (end - start !== referenceStr.length) {
return false;
}
if (start < 0 || end > testStr.length) {
return false;
}
for (var i = start; i < end; i++) {
var testCode = testStr.charCodeAt(i);
var referenceCode = referenceStr.charCodeAt(i - start);
// testCode.toLowerCase() for A..Z
if (isUppercaseLetter(testCode)) {
testCode = testCode | 32;
}
if (testCode !== referenceCode) {
return false;
}
}
return true;
}
function findWhiteSpaceStart(source, offset) {
for (; offset >= 0; offset--) {
if (!isWhiteSpace(source.charCodeAt(offset))) {
break;
}
}
return offset + 1;
}
function findWhiteSpaceEnd(source, offset) {
for (; offset < source.length; offset++) {
if (!isWhiteSpace(source.charCodeAt(offset))) {
break;
}
}
return offset;
}
function findDecimalNumberEnd(source, offset) {
for (; offset < source.length; offset++) {
if (!isDigit(source.charCodeAt(offset))) {
break;
}
}
return offset;
}
// § 4.3.7. Consume an escaped code point
function consumeEscaped(source, offset) {
// It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and
// that the next input code point has already been verified to be part of a valid escape.
offset += 2;
// hex digit
if (isHexDigit(getCharCode(source, offset - 1))) {
// Consume as many hex digits as possible, but no more than 5.
// Note that this means 1-6 hex digits have been consumed in total.
for (var maxOffset = Math.min(source.length, offset + 5); offset < maxOffset; offset++) {
if (!isHexDigit(getCharCode(source, offset))) {
break;
}
}
// If the next input code point is whitespace, consume it as well.
var code = getCharCode(source, offset);
if (isWhiteSpace(code)) {
offset += getNewlineLength(source, offset, code);
}
}
return offset;
}
// §4.3.11. Consume a name
// Note: This algorithm does not do the verification of the first few code points that are necessary
// to ensure the returned code points would constitute an <ident-token>. If that is the intended use,
// ensure that the stream starts with an identifier before calling this algorithm.
function consumeName(source, offset) {
// Let result initially be an empty string.
// Repeatedly consume the next input code point from the stream:
for (; offset < source.length; offset++) {
var code = source.charCodeAt(offset);
// name code point
if (isName(code)) {
// Append the code point to result.
continue;
}
// the stream starts with a valid escape
if (isValidEscape(code, getCharCode(source, offset + 1))) {
// Consume an escaped code point. Append the returned code point to result.
offset = consumeEscaped(source, offset) - 1;
continue;
}
// anything else
// Reconsume the current input code point. Return result.
break;
}
return offset;
}
// §4.3.12. Consume a number
function consumeNumber(source, offset) {
var code = source.charCodeAt(offset);
// 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-),
// consume it and append it to repr.
if (code === 0x002B || code === 0x002D) {
code = source.charCodeAt(offset += 1);
}
// 3. While the next input code point is a digit, consume it and append it to repr.
if (isDigit(code)) {
offset = findDecimalNumberEnd(source, offset + 1);
code = source.charCodeAt(offset);
}
// 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then:
if (code === 0x002E && isDigit(source.charCodeAt(offset + 1))) {
// 4.1 Consume them.
// 4.2 Append them to repr.
code = source.charCodeAt(offset += 2);
// 4.3 Set type to "number".
// TODO
// 4.4 While the next input code point is a digit, consume it and append it to repr.
offset = findDecimalNumberEnd(source, offset);
}
// 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E)
// or U+0065 LATIN SMALL LETTER E (e), ... , followed by a digit, then:
if (cmpChar(source, offset, 101 /* e */)) {
var sign = 0;
code = source.charCodeAt(offset + 1);
// ... optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+) ...
if (code === 0x002D || code === 0x002B) {
sign = 1;
code = source.charCodeAt(offset + 2);
}
// ... followed by a digit
if (isDigit(code)) {
// 5.1 Consume them.
// 5.2 Append them to repr.
// 5.3 Set type to "number".
// TODO
// 5.4 While the next input code point is a digit, consume it and append it to repr.
offset = findDecimalNumberEnd(source, offset + 1 + sign + 1);
}
}
return offset;
}
// § 4.3.14. Consume the remnants of a bad url
// ... its sole use is to consume enough of the input stream to reach a recovery point
// where normal tokenizing can resume.
function consumeBadUrlRemnants(source, offset) {
// Repeatedly consume the next input code point from the stream:
for (; offset < source.length; offset++) {
var code = source.charCodeAt(offset);
// U+0029 RIGHT PARENTHESIS ())
// EOF
if (code === 0x0029) {
// Return.
offset++;
break;
}
if (isValidEscape(code, getCharCode(source, offset + 1))) {
// Consume an escaped code point.
// Note: This allows an escaped right parenthesis ("\)") to be encountered
// without ending the <bad-url-token>. This is otherwise identical to
// the "anything else" clause.
offset = consumeEscaped(source, offset);
}
}
return offset;
}
module.exports = {
consumeEscaped: consumeEscaped,
consumeName: consumeName,
consumeNumber: consumeNumber,
consumeBadUrlRemnants: consumeBadUrlRemnants,
cmpChar: cmpChar,
cmpStr: cmpStr,
getNewlineLength: getNewlineLength,
findWhiteSpaceStart: findWhiteSpaceStart,
findWhiteSpaceEnd: findWhiteSpaceEnd
};