index.js
12.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
/**
* Index.js
*
* a request API compatible with window.fetch
*
* All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/.
*/
import http from 'node:http';
import https from 'node:https';
import zlib from 'node:zlib';
import Stream, {PassThrough, pipeline as pump} from 'node:stream';
import {Buffer} from 'node:buffer';
import dataUriToBuffer from 'data-uri-to-buffer';
import {writeToStream, clone} from './body.js';
import Response from './response.js';
import Headers, {fromRawHeaders} from './headers.js';
import Request, {getNodeRequestOptions} from './request.js';
import {FetchError} from './errors/fetch-error.js';
import {AbortError} from './errors/abort-error.js';
import {isRedirect} from './utils/is-redirect.js';
import {FormData} from 'formdata-polyfill/esm.min.js';
import {isDomainOrSubdomain} from './utils/is.js';
import {parseReferrerPolicyFromHeader} from './utils/referrer.js';
import {
Blob,
File,
fileFromSync,
fileFrom,
blobFromSync,
blobFrom
} from 'fetch-blob/from.js';
export {FormData, Headers, Request, Response, FetchError, AbortError, isRedirect};
export {Blob, File, fileFromSync, fileFrom, blobFromSync, blobFrom};
const supportedSchemas = new Set(['data:', 'http:', 'https:']);
/**
* Fetch function
*
* @param {string | URL | import('./request').default} url - Absolute url or Request instance
* @param {*} [options_] - Fetch options
* @return {Promise<import('./response').default>}
*/
export default async function fetch(url, options_) {
return new Promise((resolve, reject) => {
// Build request object
const request = new Request(url, options_);
const {parsedURL, options} = getNodeRequestOptions(request);
if (!supportedSchemas.has(parsedURL.protocol)) {
throw new TypeError(`node-fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, '')}" is not supported.`);
}
if (parsedURL.protocol === 'data:') {
const data = dataUriToBuffer(request.url);
const response = new Response(data, {headers: {'Content-Type': data.typeFull}});
resolve(response);
return;
}
// Wrap http.request into fetch
const send = (parsedURL.protocol === 'https:' ? https : http).request;
const {signal} = request;
let response = null;
const abort = () => {
const error = new AbortError('The operation was aborted.');
reject(error);
if (request.body && request.body instanceof Stream.Readable) {
request.body.destroy(error);
}
if (!response || !response.body) {
return;
}
response.body.emit('error', error);
};
if (signal && signal.aborted) {
abort();
return;
}
const abortAndFinalize = () => {
abort();
finalize();
};
// Send request
const request_ = send(parsedURL.toString(), options);
if (signal) {
signal.addEventListener('abort', abortAndFinalize);
}
const finalize = () => {
request_.abort();
if (signal) {
signal.removeEventListener('abort', abortAndFinalize);
}
};
request_.on('error', error => {
reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, 'system', error));
finalize();
});
fixResponseChunkedTransferBadEnding(request_, error => {
response.body.destroy(error);
});
/* c8 ignore next 18 */
if (process.version < 'v14') {
// Before Node.js 14, pipeline() does not fully support async iterators and does not always
// properly handle when the socket close/end events are out of order.
request_.on('socket', s => {
let endedWithEventsCount;
s.prependListener('end', () => {
endedWithEventsCount = s._eventsCount;
});
s.prependListener('close', hadError => {
// if end happened before close but the socket didn't emit an error, do it now
if (response && endedWithEventsCount < s._eventsCount && !hadError) {
const error = new Error('Premature close');
error.code = 'ERR_STREAM_PREMATURE_CLOSE';
response.body.emit('error', error);
}
});
});
}
request_.on('response', response_ => {
request_.setTimeout(0);
const headers = fromRawHeaders(response_.rawHeaders);
// HTTP fetch step 5
if (isRedirect(response_.statusCode)) {
// HTTP fetch step 5.2
const location = headers.get('Location');
// HTTP fetch step 5.3
let locationURL = null;
try {
locationURL = location === null ? null : new URL(location, request.url);
} catch {
// error here can only be invalid URL in Location: header
// do not throw when options.redirect == manual
// let the user extract the errorneous redirect URL
if (request.redirect !== 'manual') {
reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'));
finalize();
return;
}
}
// HTTP fetch step 5.5
switch (request.redirect) {
case 'error':
reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect'));
finalize();
return;
case 'manual':
// Nothing to do
break;
case 'follow': {
// HTTP-redirect fetch step 2
if (locationURL === null) {
break;
}
// HTTP-redirect fetch step 5
if (request.counter >= request.follow) {
reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect'));
finalize();
return;
}
// HTTP-redirect fetch step 6 (counter increment)
// Create a new Request object.
const requestOptions = {
headers: new Headers(request.headers),
follow: request.follow,
counter: request.counter + 1,
agent: request.agent,
compress: request.compress,
method: request.method,
body: clone(request),
signal: request.signal,
size: request.size,
referrer: request.referrer,
referrerPolicy: request.referrerPolicy
};
// when forwarding sensitive headers like "Authorization",
// "WWW-Authenticate", and "Cookie" to untrusted targets,
// headers will be ignored when following a redirect to a domain
// that is not a subdomain match or exact match of the initial domain.
// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
// will forward the sensitive headers, but a redirect to "bar.com" will not.
if (!isDomainOrSubdomain(request.url, locationURL)) {
for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) {
requestOptions.headers.delete(name);
}
}
// HTTP-redirect fetch step 9
if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream.Readable) {
reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect'));
finalize();
return;
}
// HTTP-redirect fetch step 11
if (response_.statusCode === 303 || ((response_.statusCode === 301 || response_.statusCode === 302) && request.method === 'POST')) {
requestOptions.method = 'GET';
requestOptions.body = undefined;
requestOptions.headers.delete('content-length');
}
// HTTP-redirect fetch step 14
const responseReferrerPolicy = parseReferrerPolicyFromHeader(headers);
if (responseReferrerPolicy) {
requestOptions.referrerPolicy = responseReferrerPolicy;
}
// HTTP-redirect fetch step 15
resolve(fetch(new Request(locationURL, requestOptions)));
finalize();
return;
}
default:
return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`));
}
}
// Prepare response
if (signal) {
response_.once('end', () => {
signal.removeEventListener('abort', abortAndFinalize);
});
}
let body = pump(response_, new PassThrough(), error => {
if (error) {
reject(error);
}
});
// see https://github.com/nodejs/node/pull/29376
/* c8 ignore next 3 */
if (process.version < 'v12.10') {
response_.on('aborted', abortAndFinalize);
}
const responseOptions = {
url: request.url,
status: response_.statusCode,
statusText: response_.statusMessage,
headers,
size: request.size,
counter: request.counter,
highWaterMark: request.highWaterMark
};
// HTTP-network fetch step 12.1.1.3
const codings = headers.get('Content-Encoding');
// HTTP-network fetch step 12.1.1.4: handle content codings
// in following scenarios we ignore compression support
// 1. compression support is disabled
// 2. HEAD request
// 3. no Content-Encoding header
// 4. no content response (204)
// 5. content not modified response (304)
if (!request.compress || request.method === 'HEAD' || codings === null || response_.statusCode === 204 || response_.statusCode === 304) {
response = new Response(body, responseOptions);
resolve(response);
return;
}
// For Node v6+
// Be less strict when decoding compressed responses, since sometimes
// servers send slightly invalid responses that are still accepted
// by common browsers.
// Always using Z_SYNC_FLUSH is what cURL does.
const zlibOptions = {
flush: zlib.Z_SYNC_FLUSH,
finishFlush: zlib.Z_SYNC_FLUSH
};
// For gzip
if (codings === 'gzip' || codings === 'x-gzip') {
body = pump(body, zlib.createGunzip(zlibOptions), error => {
if (error) {
reject(error);
}
});
response = new Response(body, responseOptions);
resolve(response);
return;
}
// For deflate
if (codings === 'deflate' || codings === 'x-deflate') {
// Handle the infamous raw deflate response from old servers
// a hack for old IIS and Apache servers
const raw = pump(response_, new PassThrough(), error => {
if (error) {
reject(error);
}
});
raw.once('data', chunk => {
// See http://stackoverflow.com/questions/37519828
if ((chunk[0] & 0x0F) === 0x08) {
body = pump(body, zlib.createInflate(), error => {
if (error) {
reject(error);
}
});
} else {
body = pump(body, zlib.createInflateRaw(), error => {
if (error) {
reject(error);
}
});
}
response = new Response(body, responseOptions);
resolve(response);
});
raw.once('end', () => {
// Some old IIS servers return zero-length OK deflate responses, so
// 'data' is never emitted. See https://github.com/node-fetch/node-fetch/pull/903
if (!response) {
response = new Response(body, responseOptions);
resolve(response);
}
});
return;
}
// For br
if (codings === 'br') {
body = pump(body, zlib.createBrotliDecompress(), error => {
if (error) {
reject(error);
}
});
response = new Response(body, responseOptions);
resolve(response);
return;
}
// Otherwise, use response as-is
response = new Response(body, responseOptions);
resolve(response);
});
// eslint-disable-next-line promise/prefer-await-to-then
writeToStream(request_, request).catch(reject);
});
}
function fixResponseChunkedTransferBadEnding(request, errorCallback) {
const LAST_CHUNK = Buffer.from('0\r\n\r\n');
let isChunkedTransfer = false;
let properLastChunkReceived = false;
let previousChunk;
request.on('response', response => {
const {headers} = response;
isChunkedTransfer = headers['transfer-encoding'] === 'chunked' && !headers['content-length'];
});
request.on('socket', socket => {
const onSocketClose = () => {
if (isChunkedTransfer && !properLastChunkReceived) {
const error = new Error('Premature close');
error.code = 'ERR_STREAM_PREMATURE_CLOSE';
errorCallback(error);
}
};
const onData = buf => {
properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
// Sometimes final 0-length chunk and end of message code are in separate packets
if (!properLastChunkReceived && previousChunk) {
properLastChunkReceived = (
Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0
);
}
previousChunk = buf;
};
socket.prependListener('close', onSocketClose);
socket.on('data', onData);
request.on('close', () => {
socket.removeListener('close', onSocketClose);
socket.removeListener('data', onData);
});
});
}