node.js - 使用gzip / deflate压缩轻松进行http请求

6 回答

• 3

  查看source code内部 - 您必须在请求lib本身上设置 gzip 参数才能使gzip正常工作 . 不确定这是否是故意的，但这是当前的实施 . 不需要额外的 Headers .

  var request = require('request');
  request.gzip = true;
  request({url: 'https://...'},  // use encoding:null for buffer instead of UTF8
      function(error, response, body) { ... }
  );
  

  回复于 2024-05-05T07:44:02+08:00
• 1

  您只需将 request 和 zlib 与流组合即可 .

  这是一个假设您有一个服务器侦听端口8000的示例：

  var request = require('request'), zlib = require('zlib');
  
      var headers = {
        'Accept-Encoding': 'gzip'
      };
  
      request({url:'http://localhost:8000/', 'headers': headers})
          .pipe(zlib.createGunzip()) // unzip
          .pipe(process.stdout); // do whatever you want with the stream
  

  回复于 2024-05-05T07:44:02+08:00
• 80

  对于最近遇到这种情况的人来说，请求库现在支持开箱即用的gzip解压缩 . 使用方法如下：

  request(
      { method: 'GET'
      , uri: 'http://www.google.com'
      , gzip: true
      }
    , function (error, response, body) {
        // body is the decompressed response body
        console.log('server encoded the data as: ' + (response.headers['content-encoding'] || 'identity'))
        console.log('the decoded data is: ' + body)
      }
    )
  

  来自github自述文件https://github.com/request/request

  gzip - 如果为true，则添加Accept-Encoding标头以从服务器请求压缩内容编码（如果尚未存在）并解码响应中支持的内容编码 . 注意：对通过请求返回的正文数据（通过请求流并传递给回调函数）执行响应内容的自动解码，但不对响应流（可从响应事件获得）执行，这是未修改的http .IncomingMessage对象，可能包含压缩数据 . 见下面的例子 .

  回复于 2024-05-05T07:44:02+08:00
• 1

  这是一个枪击响应的工作示例

  function gunzipJSON(response){
  
      var gunzip = zlib.createGunzip();
      var json = "";
  
      gunzip.on('data', function(data){
          json += data.toString();
      });
  
      gunzip.on('end', function(){
          parseJSON(json);
      });
  
      response.pipe(gunzip);
  }
  

  完整代码：https://gist.github.com/0xPr0xy/5002984

  回复于 2024-05-05T07:44:02+08:00
• 5

  查看http://nodejs.org/docs/v0.6.0/api/zlib.html#examples的示例

  zlib现在内置于节点中 .

  回复于 2024-05-05T07:44:02+08:00
• 61

  //functions.js:
  var ce=require('cloneextend');
  //console.log({aa:'bb',dd:new Date('10/10/2011')});
  //console.log(ce.clone({aa:'bb',dd:new Date('10/10/2011')}));
  exports.cloneextend=ce;
  exports.clone=ce.clone;
  exports.extend=ce.extend;
      ////////////request
  var request1=require('request');
  var Iconv=require('iconv').Iconv;
  var iconv_utf8_to_latin = new Iconv('utf-8','iso-8859-1');
  var iconv_iso8859_8i_to_utf8 = new Iconv('iso-8859-8','utf-8');
  var iconv_utf8_to_iso8859_8i = new Iconv('utf-8','iso-8859-8');
  exports.iconv_iso8859_8i_to_utf8=iconv_iso8859_8i_to_utf8;
  exports.iconv_utf8_to_iso8859_8i=iconv_utf8_to_iso8859_8i;
  
  var zlib=require('zlib');
  
  function request_unzip(options,cb)
  {
   var enc=options.encoding;
   options.encoding=null;
   var r=request1(options)
   .on('response',function(response)
   {
    var bufarr=[];
    var errored=false;
    switch (response.headers['content-encoding'])
    {
       // or, just use zlib.createUnzip() to handle both cases
       case 'gzip':
       case 'deflate':
         if(response.headers['content-encoding']=='gzip')
           var zpipe=zlib.createGunzip();
         else
           var zpipe=zlib.createInflate();
  
         zpipe
                   .on('data', function(d){bufarr.push(d);})           
                   .on('end', function(){ if(errored) return;errored=true; cb(null,response, enc?Buffer.concat(bufarr).toString(enc):Buffer.concat(bufarr) ); })
                   .on('error', function(err){ if(errored) return;errored=true; cb(err,response,null);});      
         response.pipe(zpipe);
         response
                   .on('error', function(err){ if(errored) return;errored=true; cb(err,response,null);});
         break;
       default:
         response
                   .on('data', function(d){bufarr.push(d);})           
                   .on('end', function(){ if(errored) return;errored=true; cb(null,response, enc?Buffer.concat(bufarr).toString(enc):Buffer.concat(bufarr) ); })
                   .on('error', function(err){ if(errored) return;errored=true; cb(err,response,null);});      
         break;
    }  
   });
   return r; 
  }
  
  function request(options,cb)// a request that fixes encoding
  {
   if(options.encoding=='iso-8859-8')
   {
    options.encoding='binary';
    return request_unzip(options, function(error,request,data){
     if(data===undefined||data===null)
     {
       data2=null;
       cb(error,request,data2);
     }
     else
     {  
       try{
         cb(error,request,
           iconv_iso8859_8i_to_utf8.convert(iconv_utf8_to_latin.convert(data)).toString('utf8') //conver buffer to string
         );
       }
       catch(e)
       {
        data2=null;
        error=e;
        cb(error,request,data2);
       } 
     }   
    });
   }
   else
    return request_unzip(options,cb);
  }
  request.__proto__=request1;
  exports.request=request;
  
  ie9headers= // no var goes to global
  {
   followAllRedirects:true,
   headers:
   {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "he-IL,he;q=0.8,en-US;q=0.6,en;q=0.4",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22",//"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)",              
    "Accept-Charset": "windows-1255,utf-8;q=0.7,*;q=0.3",
    "Accept-Encoding":    "gzip,deflate,sdch"
   }
  }
  
  ///
  example:
  
  f=require('./function.js'); //goes global
  function getsomething(cb){
  function getit(){
  f.request(f.extend({jar:j,url:myurl, headers:{Referer:url}, encoding:'UTF-8' },ie9headers),function(error,request,data)
  {
   if(error) setTimeout(getit,1000);
   //console.log("data",data);
   var parsed=myparse(data);
   cb(parsed);
  });}
  getit();
  }
  

  回复于 2024-05-05T07:44:02+08:00