fix plugins

plugins/di/di-gistda-air/job-creator.js

@@ -7,7 +7,7 @@ var port = "21";
 var user = "bs";
 var pwd = "UF13kczHdCPXpBb";
 var main_folder = "GISTDA_SOS_DATA"
-var init_observed_date = "2017-10-18";
+var init_observed_date = "2017-11-21";
 var init_observed_time = "09:00:00";
 var di_plugin = "sftp-filesync";
 var data_source = "gistda-air"

plugins/di/di-gistda-air/perform.js

@@ -46,7 +46,6 @@ function execute_function(context,response){
               } else {
                 latestDate = new Date(value);
               }
-
               async.eachSeries(
                   list,
                   function(element, callback) {
@@ -55,7 +54,6 @@ function execute_function(context,response){
                               var filename = element.name;
                               var filedate = element.date;
                               var filetype = element.type;
-//                              if ((path.extname(filename) === '.dat' || path.extname(filename) === '.jpg') && filename.indexOf("debug") == -1) {  
                               if ((path.extname(filename) === '.dat' && 
                                     (filename.indexOf("Every_5m") > 0 || (filename.indexOf("MS700") > 0 && filename.indexOf("debug") == -1)))
                                   || path.extname(filename) === '.jpg')  {  
@@ -67,13 +65,26 @@ function execute_function(context,response){
                                   if (filedate - latestDate > 0) {  // filter out old files
                                       c.get(param.path+"/"+filename, function (err, stream) {
                                           if (err) throw err;
-                                          var data = '';
-                                          stream.setEncoding('utf8');
+  
+                                          // for text only, not for binary data  
+                                          // var data = '';
+                                          // stream.setEncoding('utf8');
+                                          // console.log("downloading .... : " + filename + ", " + dateFormat(filedate, "isoDateTime")); 
+                                          // stream.on('data', function(chunk) {  // donwload each individual chunk as per a downloading file
+                                          //     if (chunk != '')
+                                          //       data = data + chunk;                                        
+                                          // });
+
+                                          var buf_data = Buffer.from('');
                                           console.log("downloading .... : " + filename + ", " + dateFormat(filedate, "isoDateTime")); 
+                                          var nb;  
                                           stream.on('data', function(chunk) {  // donwload each individual chunk as per a downloading file
-                                              if (chunk != '')
-                                                data = data + chunk;                                        
+                                              if (chunk != '') {
+                                                 var buf_chunk = Buffer.from(chunk);
+                                                 buf_data = Buffer.concat([buf_data, buf_chunk]);
+                                               }
                                           });
+
                                           stream.on('end', function () {  // insert a data file
                                               result.data.push({
                                                 "filename": filename,
@@ -82,8 +93,11 @@ function execute_function(context,response){
                                                 "longitude": profile.longitude,
                                                 "type": type,
                                                 "observeddatetime": dateFormat(filedate, 'yyyy-mm-dd HH:MM:ss'),
-                                                "value" : data
+                                                "value" : buf_data // data if text download only
                                               });
+
+                                              console.log(buf_data);
+
                                               if (typeof maxdate == 'undefined') {
                                                   maxdate = filedate;
                                               } else {

plugins/di/di-http-request/perform.js

@@ -5,6 +5,8 @@ function execute_function(context,response){
   var transaction_id = context.transaction.id;
   var param = context.jobconfig.data_in.param;
   var memstore = context.task.memstore
+  var input_data = context.input.data;
+  var input_meta = context.input.meta;
 
   var output_type = 'text';
   var url = param.url;

plugins/dt/dt-filter-duplicate/perform.js

@@ -11,8 +11,23 @@ function perform_function(context,request,response){
   var data = request.data;
   var meta = request.meta;
 
-  var prm_name = (param.name)?'dupkey-'+param.name:'dupkey';
+//keyname
+  //var prm_name = (param.name)?'dupkey-'+param.name:'dupkey';
+  var prm_name = 'dupkey';
+  if(param.name){
+    var name_env = {
+      'type' : output_type,
+      'data' : data,
+      'meta' : meta,
+      'name_posfix' : ''
+    }
+    var nscript = new vm.Script("name_posfix=`" + param.name + "`");
+    var ncontext = new vm.createContext(name_env);
+    nscript.runInContext(ncontext);
+    prm_name = 'dupkey-' + name_env.name_posfix;
+  }
 
+//keydata
   var datakey = data;
 
   if(param.key){
@@ -28,9 +43,6 @@ function perform_function(context,request,response){
     script.runInContext(context);
 
     datakey = env.key;
-    data = env.data;
-    meta = env.meta;
-    output_type = env.type;
   }
 
   var hash_key = hash(datakey);

plugins/dt/dt-gistda-air/perform.js

@@ -138,7 +138,7 @@ function perform_image(_result, filecontent) {
 
 function getImage(filecontent) {
   return  new Promise((resolve, reject) => {
-      resolve("data:image;base64," + new Buffer(filecontent).toString('base64'));
+      resolve("data:image/jpeg;base64," + new Buffer(filecontent).toString('base64'));
   });    
 }