Merge branch '2-use-objects' into 'master'

Resolve "use objects"

Closes #2

See merge request !4

src/wikidata-analyser-transformer.ts

@@ -102,8 +102,17 @@ class WikidataAnalyserTransformer extends Transform {
    */
   public _transform(chunk, encoding, callback) {
     // if norm_identifiers are unset, go ahead
-    const data = Buffer.from(chunk.parsed.data).toString('utf8');
-    const obj = JSON.parse(data);
+    let obj;
+    if(Buffer.isBuffer(chunk)) {
+      try {
+        const data = Buffer.from(chunk).toString('utf8');
+        obj = JSON.parse(data);
+      } catch(err) {
+        return callback(null);
+      }
+    } else {
+      obj = chunk;
+    }
 
     if (typeof obj.claims === 'undefined') {
       return callback(null);

src/wikidata-geofilter-transformer.ts

@@ -23,13 +23,17 @@ class WikidataGeofilterTransformer extends Transform {
    */
   public _transform(chunk, encoding, callback) {
       // check if instance of exists
-      let obj;
+    let obj;
+    if(Buffer.isBuffer(chunk)) {
       try {
-        const data = Buffer.from(chunk.parsed.data).toString('utf8');
+        const data = Buffer.from(chunk).toString('utf8');
         obj = JSON.parse(data);
-      } catch (err) {
+      } catch(err) {
         return callback(null);
       }
+    } else {
+      obj = chunk;
+    }
 
       if (typeof obj.claims === 'undefined' || typeof obj.claims.P31 === 'undefined') {
         return callback(null);

src/wikidata-normalizer-transformer.ts

@@ -40,9 +40,19 @@ class WikidataNormalizerTransformer extends Transform {
    */
   public _transform(chunk, encoding, callback) {
     let result = {};
+    let obj;
+    if(Buffer.isBuffer(chunk)) {
+      try {
+        const data = Buffer.from(chunk).toString('utf8');
+        obj = JSON.parse(data);
+      } catch(err) {
+        return callback(null);
+      }
+    } else {
+      obj = chunk;
+    }
+
     try {
-      const data = Buffer.from(chunk.parsed.data).toString('utf8');
-      const obj = JSON.parse(data);
       // helpers
       const lat = Number.parseFloat(this.getValue(obj, 'P625', 'latitude')).toFixed(6);
       const lon = Number.parseFloat(this.getValue(obj, 'P625', 'longitude')).toFixed(6);

test/wikidata-analyser-transformer.spec.ts

@@ -41,11 +41,59 @@ describe('Test wikidata-analyser-transformer', () => {
     });
 
     // build chunk
-    const chunck = {
-      parsed: {
-        data: Buffer.from(fs.readFileSync(__dirname + '/fixtures/place.json')),
-      },
+    const chunck = Buffer.from(fs.readFileSync(__dirname + '/fixtures/place.json'))
+    analyser.write(chunck);
+  });
+
+  it('Try to transform json-string data', (done) => {
+    const options = {
+      objectMode: true,
+      nconf,
+      uriBuilder: coreMock.getUriBuilder(),
     };
+    const analyser = new WikidataAnalyserTransformer(options);
+
+    // listener to check output
+    analyser.on('data', (data) => {
+      const testMessage = {
+        from: 'https://www.wikidata.org/wiki/Q13038',
+        to:
+          ['https://g.co/kg/m/0gdcn4', 'http://geonames.org/8261346'],
+        relation: {type: 'same_as', author: undefined, weight: 1},
+      };
+
+      data.should.be.deep.equal(testMessage);
+      done();
+    });
+
+    // build chunk
+    const chunck = fs.readFileSync(__dirname + '/fixtures/place.json');
     analyser.write(chunck);
   });
+
+  it('Try to transform json-object data', (done) => {
+    const options = {
+      objectMode: true,
+      nconf,
+      uriBuilder: coreMock.getUriBuilder(),
+    };
+    const analyser = new WikidataAnalyserTransformer(options);
+
+    // listener to check output
+    analyser.on('data', (data) => {
+      const testMessage = {
+        from: 'https://www.wikidata.org/wiki/Q13038',
+        to:
+          ['https://g.co/kg/m/0gdcn4', 'http://geonames.org/8261346'],
+        relation: {type: 'same_as', author: undefined, weight: 1},
+      };
+
+      data.should.be.deep.equal(testMessage);
+      done();
+    });
+
+    // build chunk
+    const chunck: string = fs.readFileSync(__dirname + '/fixtures/place.json', 'utf8');
+    analyser.write(JSON.parse(chunck));
+  });
 });

test/wikidata-geofilter-transformer.spec.ts

@@ -15,12 +15,7 @@ describe('Test wikidata-geofilter-transformer', () => {
   it('Try to successfully identify a location', (done) => {
     const mock = fs.readFileSync(__dirname + '/fixtures/place.json');
     // build chunk
-    const chunk = {
-      parsed: {
-        data: Buffer.from(mock),
-      },
-    };
-
+    const chunk =  Buffer.from(mock);
     // Q1840161 = commune of chile a subclass of location
     const geoInstances = ['Q1840161'];
     const options = {
@@ -41,11 +36,7 @@ describe('Test wikidata-geofilter-transformer', () => {
   it('Try to filter out a non location item (person)', (done) => {
     const mock = fs.readFileSync(__dirname + '/fixtures/person.json');
     // build chunk
-    const chunk = {
-      parsed: {
-        data: Buffer.from(mock),
-      },
-    };
+    const chunk =  Buffer.from(mock);
 
     // Q1840161 = commune of chile a subclass of location
     const geoInstances = ['Q1840161'];

test/wikidata-normalizer-transformer.spec.ts

@@ -48,11 +48,7 @@ describe('Test WikiNormalizer', () => {
     });
 
     // build chunk
-    const chunck = {
-      parsed: {
-        data: Buffer.from(fs.readFileSync(__dirname + '/fixtures/place.json')),
-      },
-    };
+    const chunck = Buffer.from(fs.readFileSync(__dirname + '/fixtures/place.json'))
     normalizer.write(chunck);
     normalizer.end();
   });