服务器崩溃时,使用NodeJS从url下载大文件
问题描述:
我试图从使用来自nodejs的请求模块的url(需要登录)下载文件(+ 200mb),但是当它完成下载时服务器开始减速直到崩溃或变得非常缓慢。服务器崩溃时,使用NodeJS从url下载大文件
这里是我当前的代码(它下载整个文件,但最终我的服务器崩溃):
//Required modules
var http = require('http'),
url = require("url"),
fs = require('fs'),
request = require('request'),
path = require("path"),
events = require("events"),
j = request.jar(),
request = request.defaults({ jar : j });
// make te request login in with cookies
console.log("downloading file :)");
request({
url:"http://example.com/",
method:"POST",
form:{u: "username",p: "password"}
},
function(error,response,body){
setTimeout(function(){
request
.get('http://example.com/test.ashx?file=15')
.on('error', function(err) {
console.log(err);
})
.pipe(fs.createWriteStream("/var/www/filesDir/CustomName.zip"));
console.log(body);
},1000)
}
);
我试着从this answer应用另一种解决方案,但由于某些原因,文件没有正确下载,它总是只显示“下载进度:0字节”,可能与登录访问有关。
这里我把其他的代码我试图从最后一句来实现:
var http = require('http');
var fs = require('fs');
var url = require("url");
var request = require('request');
var path = require("path");
var events = require("events");
var j = request.jar();
var request = request.defaults({ jar : j });
request({
url:"http://example.com/",
method:"POST",
form:{u:"username",p:"password"}
}, function(error,response,body){
var downloadfile = "http://example.com/test.ashx?file=15";
var host = url.parse(downloadfile).hostname;
var filename = "1977.zip";
var req = http.request({port: 80, host: host, method: 'GET'});
console.log("Downloading file: " + filename);
console.log("Before download request");
req.end();
dlprogress = 0;
setInterval(function() {
console.log("Download progress: " + dlprogress + " bytes");
}, 1000);
req.addListener('response', function (response) {
var downloadfile = fs.createWriteStream(filename, {'flags': 'a'});
console.log("File size " + filename + ": " + response.headers['content-length'] + " bytes.");
response.addListener('data', function (chunk) {
dlprogress += chunk.length;
downloadfile.write(chunk, encoding='binary');
});
response.addListener("end", function() {
downloadfile.end();
console.log("Finished downloading " + filename);
});
});
}
);
不要紧,你决定要帮我用哪种方式。
答
我最后只是这样,我测试的代码多次,服务器也不再崩溃:
var request = require('request');
var filed = require('filed');
var j = request.jar();
var request = request.defaults({ jar : j });
// make the request and login
request({
url: "http://example.com/login",
method:"POST",
// 'u' and 'p' are the field names on the form
form:{u:"username",p:"password"}
}, function(error,response,body){
setTimeout(function(){
var downloadURL = 'http://example.com/download/file.zip';
var downloadPath = "/path/to/download/localNameForFile.zip";
var downloadFile = filed(downloadPath);
var r = request(downloadURL).pipe(downloadFile);
r.on('data', function(data) {
console.log('binary data received');
});
downloadFile.on('end', function() {
console.log(downloadPath, 'file downloaded to path');
});
downloadFile.on('error', function (err) {
console.log(err, 'error downloading file');
});
},3000)
}
);