update

lib/bss/objectdata.js

@@ -57,7 +57,8 @@ ObjectData.prototype.readMeta = function(opt,cb){
 
   fd.bufferedRead(oStart,oLen,function(err,buff){
     if(!err){
-      cb(null,BSON.parse(buff));
+      //cb(null,BSON.parse(buff));
+      cb(null,safe_parse_bson(buff));
     }else{
       cb(err);
     }
@@ -104,7 +105,8 @@ ObjectData.prototype.readObject = function(opt,cb){
     if(!err){
       var obj = {
         header : self.header,
-        meta : BSON.parse(buff.slice(0,self.header.MZ)),
+        //meta : BSON.parse(buff.slice(0,self.header.MZ)),
+        meta : safe_parse_bson(buff.slice(0,self.header.MZ)),
         data : parse_data(buff.slice(self.header.MZ,self.header.MZ+self.header.DZ),self.header.TY),
       }
 
@@ -120,12 +122,24 @@ var parse_data = function(buffer,ty){
   if(ty==STRING_TYPE){
     return buffer.toString('utf8');
   }else if(ty==OBJECT_TYPE){
-    return BSON.parse(buffer);
+    //return BSON.parse(buffer);
+    return safe_parse_bson(buffer);
   }else{
     return buffer;
   }
 }
 
+var safe_parse_bson = function(buffer)
+{
+  var ret={};
+  try{
+    ret=BSON.parse(buffer);
+  }catch(err){
+    ret={};
+  }
+  return ret;
+}
+
 ObjectData.prototype.write = function(addr,cb){
   var address = this.header.AD;
 

plugins/dt/dt-filter-duplicate/perform.js

@@ -37,6 +37,7 @@ function perform_function(context,request,response){
   memstore.getItem(prm_name,function(err,value){
     if(value && value==hash_key)
     {
+      memstore.setItem(prm_name,hash_key,function(err){})
       response.reject();
     }else{
       memstore.setItem(prm_name,hash_key,function(err){})

plugins/dt/dt-gistda-water/index.js

@@ -4,7 +4,7 @@ var DTPlugin = require('../dt-plugin');
 
 function DTTask(context,request){
   DTPlugin.call(this,context,request);
-  this.name = "noop";
+  this.name = "gistda-water";
   this.output_type = "";
 }
 util.inherits(DTTask,DTPlugin);

test/bss_read.js

@@ -3,7 +3,7 @@ var async = require('async');
 
 var BinStream = ctx.getLib('lib/bss/binarystream_v1_1');
 
-var FNAME = "D:/testfile/env.bss";
+var FNAME = "D:/testfile/agritronics.bss";
 
 
 // BinStream.open(FNAME,function(err,bss){
@@ -54,10 +54,13 @@ BinStream.open(FNAME,function(err,bss){
             cont=false;
             callback();
           }else{
-            obj.readMeta(function(err,meta){
+            obj.readMeta(function(err,obj){
               idx++;
-              if(idx%100000 == 0){
-                console.log(meta);
+              if(idx%10000 == 0){
+                console.log(obj);
+              }
+              if(idx==86639){
+                console.log(obj);
               }
               callback();
             });