我们可以使用IBM Cloud Object Storage保存文件,IBM Cloud Object Storage提供了一系列的SDK,方便我们操作Storage。比如Java, Nodejs等等。
官方文档:https://cloud.ibm.com/docs/cloud-object-storage/libraries?topic=cloud-object-storage-node
下面是官方给的Nodejs上传大文件的Sample代码:
这个代码实际使用时,对于2GB以上的大文件,会出错。原因是 fs.readFile 这个方法本身不支持2GB以上的文件,因为这个方法会把文件读到内存中,如果文件过大,会造成内存溢出。
function multiPartUpload(bucketName, itemName, filePath) {
var uploadID = null;
if (!fs.existsSync(filePath)) {
log.error(new Error(`The file \'${filePath}\' does not exist or is not accessible.`));
return;
}
console.log(`Starting multi-part upload for ${itemName} to bucket: ${bucketName}`);
return cos.createMultipartUpload({
Bucket: bucketName,
Key: itemName
}).promise()
.then((data) => {
uploadID = data.UploadId;
//begin the file upload
// 注意!!! fs.readFile方法不支持2GB以上的文件,因为fs.readFile会把文件读取到内存中,当文件过大时,是不允许的。
fs.readFile(filePath, (e, fileData) => {
//min 5MB part
var partSize = 1024 * 1024 * 5;
var partCount = Math.ceil(fileData.length / partSize);
async.timesSeries(partCount, (partNum, next) => {
var start = partNum * partSize;
var end = Math.min(start + partSize, fileData.length);
partNum++;
console.log(`Uploading to ${itemName} (part ${partNum} of ${partCount})`);
cos.uploadPart({
Body: fileData.slice(start, end),
Bucket: bucketName,
Key: itemName,
PartNumber: partNum,
UploadId: uploadID
}).promise()
.then((data) => {
next(e, {ETag: data.ETag, PartNumber: partNum});
})
.catch((e) => {
cancelMultiPartUpload(bucketName, itemName, uploadID);
console.error(`ERROR: ${e.code} - ${e.message}\n`);
});
}, (e, dataPacks) => {
cos.completeMultipartUpload({
Bucket: bucketName,
Key: itemName,
MultipartUpload: {
Parts: dataPacks
},
UploadId: uploadID
}).promise()
.then(console.log(`Upload of all ${partCount} parts of ${itemName} successful.`))
.catch((e) => {
cancelMultiPartUpload(bucketName, itemName, uploadID);
console.error(`ERROR: ${e.code} - ${e.message}\n`);
});
});
});
})
.catch((e) => {
console.error(`ERROR: ${e.code} - ${e.message}\n`);
});
}
function cancelMultiPartUpload(bucketName, itemName, uploadID) {
return cos.abortMultipartUpload({
Bucket: bucketName,
Key: itemName,
UploadId: uploadID
}).promise()
.then(() => {
console.log(`Multi-part upload aborted for ${itemName}`);
})
.catch((e)=>{
console.error(`ERROR: ${e.code} - ${e.message}\n`);
});
}

下面是修改后的代码,用fs.openSync代替了 fs.readFile
function uploadItem(itemName, filePath) {
var p = new Promise((resolve, reject) => {
let uploadID = null;
if (!fs.existsSync(filePath)) {
reject(itemName)
return;
}
cos.createMultipartUpload({
Bucket: environment.bucketName,
Key: itemName,
}).promise()
.then((data) => {
uploadID = data.UploadId;
fs.readFile("server.js", (e, fileData) => {
// 这里使用fs.statSync来获得文件size
var stats = fs.statSync(filePath);
// 这里使用fs.openSync来打开大文件
var fd = fs.openSync(filePath, 'r');
// min 40MB part
const partSize = 1024 * 1024 * 40;
const partCount = Math.ceil(stats.size / partSize);
async.timesSeries(partCount, (partNum, next) => {
// 计算每次的游标开始位置
const start = partNum * partSize;
// 计算每次的游标终了位置
const end = Math.min(start + partSize, stats.size);
// 这里使用按游标和字节数来读取文件。
var bufferRead = Buffer.alloc(end - start);
fs.readSync(fd, bufferRead, 0, end - start, start);
partNum++;
cos.uploadPart({
Body: bufferRead,
Bucket: environment.bucketName,
Key: itemName,
PartNumber: partNum,
UploadId: uploadID,
}).promise()
.then((data) => {
next(e, {ETag: data.ETag, PartNumber: partNum});
})
.catch((e) => {
fs.close(fd);
reject(itemName)
});
}, (e, dataPacks) => {
// 关闭文件
fs.close(fd);
// 完成上传
cos.completeMultipartUpload({
Bucket: environment.bucketName,
Key: itemName,
MultipartUpload: {
Parts: dataPacks,
},
UploadId: uploadID,
}).promise()
// 删除文件
fs.unlink(filePath, function (err) {
if (err) {
reject(itemName)
return;
}
resolve(itemName)
})
});
});
})
.catch((e) => {
fs.close(fd);
reject(itemName);
});
});
return p;
}