/*
* Author: Vlad Seryakov vseryakov@gmail.com
* backendjs 2018
*/
'use strict';
const { basename } = require("node:path");
const qs = require('node:querystring');
const http = require('node:http');
const https = require('node:https');
const stream = require("node:stream");
const logger = require(__dirname + '/../logger');
const lib = require(__dirname + '/../lib');
const aws = require(__dirname + '/../aws');
/**
* Sign an S3 request and return the URL to send to the S3 server. `options.headers` is updated in place
* with all headers required for signing.
* @memberof module:aws
* @method signS3
* @param {string} [method=GET] - HTTP method
* @param {string} bucket - S3 bucket name
* @param {string} path - object key/path within the bucket
* @param {string|Buffer} body - request payload used for signing
* @param {object} [options]
* @param {string} [options.region] - AWS region, defaults to `aws.region` or `us-east-1`
* @param {object} [options.headers] - request headers, updated in place with signing headers
* @param {object} [options.query] - query parameters to include in the signed URL
* @param {object} [options.credentials] - credentials to sign with, defaults to `aws`
* @param {boolean} [options.nodns] - disable DNS compatibility access
* @param {boolean} [options.url] - if set, append the signing params to the URL (presigned URL)
* @param {number} [options.expires] - expiration seconds for a presigned URL
* @param {boolean} [options.postfile] - payload is a file, use UNSIGNED-PAYLOAD
* @param {string} [options.endpoint] - full endpoint url, only protocol and host are used
* @param {string} [options.endpoint_host] - custom host, default is AWS region
* @param {string} [options.endpoint_protocol] - http or https default
* @returns {string} the signed request URL
*/
aws.signS3 = function(method, bucket, path, body, options)
{
if (!options) options = {};
if (!options.headers) options.headers = {};
const region = options.region || this.region || 'us-east-1';
if (!options.headers["content-type"]) {
options.headers["content-type"] = "binary/octet-stream";
}
// For text files to prevent encoding/decoding issues
if (/text|html|xml|json/.test(options.headers["content-type"]) &&
!options.headers["content-type"].includes("charset=")) {
options.headers["content-type"] += "; charset=utf-8";
}
const credentials = options.credentials || this;
// Run through the encoding so our signature match the real url sent by app.fetch
if (typeof path !== "string") path = String(path);
if (path[0] !== "/") path = "/" + path;
const key = URL.parse("file:" + (path || "/"))?.pathname;
// DNS compatible or not, use path-style if not for access otherwise virtual host style
const dns = options.nodns ? false : /[a-z0-9][a-z0-9-]*[a-z0-9]/.test(bucket);
let host = options.endpoint_host;
let proto = lib.isString(options.endpoint_protocol, "https:");
if (options.endpoint) {
const e = URL.parse(options.endpoint);
if (e) {
host = e.host;
proto = e.protocol;
}
}
if (!proto.endsWith(":")) {
proto += ":";
}
if (!host) {
host = (dns ? bucket + "." : "") + "s3" + (region !== "us-east-1" ? "-" + region : "") + ".amazonaws.com";
}
bucket = bucket ? "/" + bucket : "";
let uri = proto + "//" + host + (dns ? "" : bucket) + key;
if (!dns) {
path = bucket + path;
}
const q = lib.objKeys(options.query).
sort().
map((x) => (aws.uriEscape(x) + (options.query[x] === null ? "" : "=" + aws.uriEscape(options.query[x])))).
join("&");
if (q) {
path += "?" + q;
}
if (!options.headers["x-amz-content-sha256"]) {
options.headers["x-amz-content-sha256"] = method !== "GET" && options.postfile ? "UNSIGNED-PAYLOAD" : lib.hash(body || "", "sha256", "hex");
}
const opts = {};
this.signQuery(region, "s3", host, method || "GET", path, body, options.headers, options.credentials, opts);
if (options.url) {
uri += '&X-Amz-Date=' + opts.date;
uri += '&X-Amz-Algorithm=AWS4-HMAC-SHA256';
uri += '&X-Amz-Credential=' + opts.credential;
uri += '&X-Amz-SignedHeaders=' + opts.signedHeaders;
uri += '&X-Amz-Signature=' + opts.signature;
if (options.expires) uri += "&X-Amz-Expires=" + options.expires;
if (credentials.token) uri += '&X-Amz-Security-Token=' + credentials.token;
}
logger.debug('signS3:', uri, options, "opts:", opts);
return uri;
}
/**
* Make a low-level signed S3 request.
* @memberof module:aws
* @method queryS3
* @param {string} bucket - S3 bucket name
* @param {string} path - object key/path within the bucket
* @param {object} [options]
* @param {string} [options.method] - HTTP method
* @param {object} [options.query] - query parameters for the URL
* @param {string|Buffer} [options.postdata] - data to send with POST/PUT
* @param {string} [options.postfile] - path of a file to upload
* @param {string} [options.poststream] - stream to upload
* @param {number} [options.expires] - absolute time when the request expires
* @param {object} [options.headers] - HTTP headers to send
* @param {string} [options.file] - file name to save the downloaded contents to
* @param {function} callback - `(err, request)`
*/
aws.queryS3 = function(bucket, path, options, callback)
{
if (typeof options === "function") callback = options, options = null;
options = lib.clone(options);
if (!options.retryCount) options.retryCount = 3;
if (!options.retryTimeout) options.retryTimeout = 1000;
options.retryOnError = function() { return this.status === 503 || this.status === 500 }
const url = this.signS3(options.method, bucket, path, options.postdata, options);
lib.fetch(url, options, (err, rc) => {
if ((rc.status < 200 || rc.status > 399) && rc.data) err = aws.parseError(rc);
if (err) err.path = path;
logger.logger(err ? "error" : "debug", "queryS3:", bucket, path, options, err);
lib.tryCall(callback, err, rc);
});
}
/**
* Async version of {@link module:aws.queryS3}
* @return {object} - result as an object `{ ok, err, status, data, obj, request }` compatible with {@link module:lib.afetch}
* @async
* @memberof module:aws
* @method aqueryS3
*/
aws.aqueryS3 = function(bucket, path, options)
{
return new Promise((resolve, _reject) => {
aws.queryS3(bucket, path, options, (err, request) => {
resolve({ err, ok: request.ok, status: request.status, data: request.data, obj: request.obj, request })
});
});
}
/**
* Retrieve a list of files from an S3 bucket, only objects under the given path are returned.
* @memberof module:aws
* @method s3List
* @param {string} path - S3 URL/path, root is the bucket (e.g. `s3://bucket/prefix` or `bucket/prefix`)
* @param {object} [options]
* @param {object} [options.query] - extra S3 list query parameters (e.g. `delimiter`)
* @param {function} callback - `(err, rows, prefixes)`
*/
aws.s3List = function(path, options, callback)
{
if (typeof options === "function") callback = options, options = {};
options = lib.clone(options);
options.query = options.query || {};
const s3 = aws.s3ParseUrl(path);
if (s3.endpoint) options.endpoint = s3.endpoint;
if (s3.path) options.query.prefix = s3.path;
if (s3.key) options.credentials = { key: s3.key, secret: s3.secret };
if (s3.bucket) options.query['list-type'] = 2;
for (const p in s3.query) options.query[p] = s3.query[p];
const rows = [], prefixes = [];
let truncated = false;
lib.doWhilst(
function(next) {
aws.queryS3(s3.bucket, "", options, (err, rc) => {
if (err) return next(err);
if (s3.bucket) {
rows.push.apply(rows, lib.objGet(rc.obj, "ListBucketResult.Contents", { list: 1 }));
prefixes.push.apply(prefixes, lib.objGet(rc.obj, "ListBucketResult.CommonPrefixes", { list: 1 }).map((x) => (x.Prefix.replace(s3.path, ""))));
truncated = lib.toBool(rc.obj?.ListBucketResult?.IsTruncated);
options.query['continuation-token'] = rc.obj?.ListBucketResult?.NextContinuationToken || "";
} else {
rows.push.apply(rows, lib.objGet(rc.obj, "ListAllMyBucketsResult.Buckets.Bucket", { list: 1 }));
truncated = rc.obj?.ListAllMyBucketsResult?.ContinuationToken;
options.query['continuation-token'] = truncated;
}
next(err);
});
},
function() {
return truncated;
}, (err) => {
lib.tryCall(callback, err, rows, prefixes);
}, true);
}
/**
* Retrieve a file from an S3 bucket; the root of the path is the bucket, an `s3://` prefix is ignored.
* @memberof module:aws
* @method s3GetFile
* @param {string} path - S3 URL/path (e.g. `s3://bucket/key`)
* @param {object} [options] - see {@link module:aws.queryS3}, e.g. `file` to save to disk
* @param {function} callback - `(err, request)`
*/
aws.s3GetFile = function(path, options, callback)
{
if (typeof options === "function") callback = options, options = {};
options = lib.clone(options);
const s3 = this.s3ParseUrl(path);
if (s3.endpoint) options.endpoint = s3.endpoint;
if (s3.query) options.query = s3.query;
if (s3.key) options.credentials = { key: s3.key, secret: s3.secret };
this.queryS3(s3.bucket, s3.path, options, (err, rc) => {
if (rc.status === 200) {
// A weird case when noSuchKey returned with status 200 as JSON
if (/^"?<\?xml version=[\s\S]+<Error><Code>/.test(rc.data)) {
if (rc.data[0] === '"') rc.data = lib.jsonParse(rc.data);
if (!rc.obj) rc.obj = lib.xmlParse(rc.data);
err = aws.parseError(rc);
if (err) err.status = 404;
}
}
lib.tryCall(callback, err, rc);
});
}
/**
* Upload a file to an S3 bucket.
* @memberof module:aws
* @method s3PutFile
* @param {string} path - S3 URL/path (e.g. `s3://bucket/key`)
* @param {string|Buffer|Stream} source - a Buffer with contents or a path to a local file or a stream
* @param {object} [options]
* @param {string} [options.acl] - canned ACL, sets `x-amz-acl`
* @param {string} [options.contentType] - object content type
* @param {function} callback - `(err, request)`
*/
aws.s3PutFile = function(path, source, options, callback)
{
if (typeof options === "function") callback = options, options = {};
options = lib.clone(options);
options.method = "PUT";
if (!options.headers) options.headers = {};
if (options.acl) options.headers['x-amz-acl'] = options.acl;
if (options.contentType) options.headers['content-type'] = options.contentType;
options[Buffer.isBuffer(source) ? 'postdata' :
stream.isReadable(source) ? "poststream": 'postfile'] = source;
const s3 = this.s3ParseUrl(path);
if (s3.endpoint) options.endpoint = s3.endpoint;
if (s3.query) options.query = s3.query;
if (s3.key) options.credentials = { key: s3.key, secret: s3.secret };
logger.debug("s3PutFile:", s3, options);
this.queryS3(s3.bucket, s3.path, options, callback);
}
/**
* Copy an existing S3 object to a new location.
* @memberof module:aws
* @method s3CopyFile
* @param {string} path - destination S3 URL/path
* @param {string} source - source object as `bucket/path` (an `s3://` prefix is removed)
* @param {object} [options]
* @param {string} [options.acl] - canned ACL for the copy
* @param {string} [options.contentType] - content type, defaults to the mime type of the path
* @param {function} callback - `(err, request)`
*/
aws.s3CopyFile = function(path, source, options, callback)
{
if (typeof options === "function") callback = options, options = {};
options = lib.clone(options);
options.method = "PUT";
if (!options.headers) options.headers = {};
options.headers["x-amz-copy-source"] = String(source).replace("s3://", "");
if (options.acl) options.headers['x-amz-acl'] = options.acl;
if (options.contentType) options.headers['content-type'] = options.contentType;
if (!options.headers['content-type']) options.headers['content-type'] = lib.getMimeType(path);
const s3 = this.s3ParseUrl(path);
if (s3.endpoint) options.endpoint = s3.endpoint;
if (s3.query) options.query = s3.query;
if (s3.key) options.credentials = { key: s3.key, secret: s3.secret };
logger.debug("s3CopyFile:", s3, options);
this.queryS3(s3.bucket, s3.path, options, callback);
}
/**
* Parse an S3 URL into its components, if starts with http:// or https:// then endpoint will be extracted,
* urls that start with s3://bucket are accepted for aws cli compatibility.
* @memberof module:aws
* @method s3ParseUrl
* @param {string} link - an S3 URL/path, optionally with `key:secret@` credentials and a query string
* @returns {object} `{ bucket, path, [key], [secret], [query], [endpoint] }`
*/
aws.s3ParseUrl = function(link)
{
var rc = { bucket: "", path: "" };
if (!lib.isString(link)) return rc;
// Endpoint detection
let d = link.match(/^https?:\/\/([^/]+)/);
if (d) {
rc.endpoint = d[0];
link = link.substr(d[0].length);
}
link = link.split("?");
// Remove the protocol part and leading slashes
link[0] = link[0].replace(/(^.+:\/\/|^\/+)/, "");
const path = link[0].replaceAll("//", "/").split("/");
rc.bucket = path[0];
// Access key and secret as auth
d = rc.bucket.match(/^([^:]+):([^@]+)@(.+)$/);
if (d) {
rc.key = d[1];
rc.secret = d[2];
rc.bucket = d[3];
}
rc.path = path.slice(1).join("/");
if (link[1]) rc.query = qs.parse(link[1]);
return rc;
}
/**
* Proxy (stream) an object from an S3 bucket into an existing HTTP response.
*
* Typically used to serve/download S3-hosted files through your app:
* it fetches `file` from `bucket` (optionally using request options like range/content-type/etc)
* and pipes the S3 response directly into `res`, preserving status/headers as appropriate.
*
* @memberof module:aws
* @function s3Proxy
*
* @param {http.ServerResponse} res Node.js HTTP response object to write to. The S3 object data is streamed into it.
* @param {string} path - S3 url, bucket and path
* @param {Object} [options] - Controls how the object is fetched and how the HTTP response is produced.
* @param {int} [options.httpTimeout] - cancel if no data after this timeout
* @param {Object} [options.headers] - Extra headers to send to S3 (commonly used for `Range`).
* @param {boolean} [options.attachment] - If true, sets `Content-Disposition` to attachment (usually derived from filename).
* @param {function(err)} [callback] Called when proxying finishes or fails.
* - `err` is set on any S3/stream/response error.
*/
aws.s3Proxy = function(res, path, options, callback)
{
if (typeof options === "function") callback = options, options = null;
options = lib.clone(options);
const s3 = this.s3ParseUrl(path);
if (s3.endpoint) options.endpoint = s3.endpoint;
const url = this.signS3("GET", s3.bucket, s3.path, "", options);
let called = false;
const done = (err) => {
if (called) return;
called = true;
if (err) logger.error("s3Proxy:", s3, err);
lib.tryCall(callback, err);
};
const mod = url.startsWith("https:") ? https : http;
const s3req = mod.request(url, { headers: options.headers }, (s3res) => {
if (options.attachment) {
const fname = typeof options.attachment === "string" ? options.attachment : basename(s3.path);
s3res.headers["Content-Disposition"] = "attachment; filename=" + fname;
}
res.writeHead(s3res.statusCode, s3res.headers);
stream.pipeline(s3res, res, done);
});
s3req.on("error", done);
s3req.setTimeout(lib.toNumber(options.httpTimeout, { dflt: 10000, min: 1000 }), () => {
s3req.destroy(lib.newError("Request timed out", 408, "ETIMEDOUT"));
});
s3req.end();
}