Skip to content

GitLab

Commit 918e8c44 authored by Tobinsk's avatar Tobinsk
Browse files

Merge branch '1-add-builded-lib' into 'master' 

Add builded lib

Closes #1

See merge request !3
parents b6d987cd cba438bc
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 918e8c44
......@@ -59,5 +59,4 @@ typings/
# next.js build output
.next
dist/
.idea/
\ No newline at end of file
.idea/
dist/KafkaProducer.js 0 → 100644
View file @ 918e8c44
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
class KafkaProducerWritable extends stream_1.Writable {
constructor(options = {}, producer, nconf, reporter = null, logger = null) {
super(options);
/**
* An optional reporter
* @type {null}
*/
this.reporter = null;
this.producer = producer;
this.nconf = nconf;
this.reporter = reporter;
this.logger = logger;
}
/**
* Produce a stream to kafka
* @param chunk
* @param {string} encoding
* @param {(err?: Error) => void} callback
* @private
*/
_write(chunk, encoding, callback) {
return __awaiter(this, void 0, void 0, function* () {
try {
yield this.producer.produce(this.nconf.get('producer:topics'), this.nconf.get('producer:partition'), chunk);
this.report();
return callback();
}
catch (err) {
// backpressure?
this.logger.error('error written to kafka');
return callback(err);
}
});
}
/**
* Report data to dashboard if possible
*/
report() {
if (this.reporter !== null) {
this.reporter.setDataOut(1);
}
}
}
exports.default = KafkaProducerWritable;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiS2Fma2FQcm9kdWNlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9LYWZrYVByb2R1Y2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7Ozs7Ozs7Ozs7QUFBQSxtQ0FBZ0M7QUFNaEMsMkJBQTRCLFNBQVEsaUJBQVE7SUFrQjFDLFlBQVksT0FBTyxHQUFHLEVBQUUsRUFBRSxRQUFrQixFQUFFLEtBQUssRUFBRSxXQUE4QixJQUFJLEVBQUUsU0FBeUIsSUFBSTtRQUNwSCxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7UUFQakI7OztXQUdHO1FBQ0ssYUFBUSxHQUFzQixJQUFJLENBQUM7UUFJekMsSUFBSSxDQUFDLFFBQVEsR0FBRyxRQUFRLENBQUM7UUFDekIsSUFBSSxDQUFDLEtBQUssR0FBRyxLQUFLLENBQUM7UUFDbkIsSUFBSSxDQUFDLFFBQVEsR0FBRyxRQUFRLENBQUM7UUFDekIsSUFBSSxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUM7SUFDdkIsQ0FBQztJQUVEOzs7Ozs7T0FNRztJQUNVLE1BQU0sQ0FBQyxLQUFLLEVBQUUsUUFBUSxFQUFFLFFBQVE7O1lBQzNDLElBQUk7Z0JBQ0YsTUFBTSxJQUFJLENBQUMsUUFBUSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxpQkFBaUIsQ0FBQyxFQUFFLElBQUksQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLG9CQUFvQixDQUFDLEVBQUUsS0FBSyxDQUFDLENBQUM7Z0JBQzVHLElBQUksQ0FBQyxNQUFNLEVBQUUsQ0FBQztnQkFDZCxPQUFPLFFBQVEsRUFBRSxDQUFDO2FBQ25CO1lBQUMsT0FBTyxHQUFHLEVBQUU7Z0JBQ1osZ0JBQWdCO2dCQUNoQixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyx3QkFBd0IsQ0FBQyxDQUFDO2dCQUM1QyxPQUFPLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQzthQUN0QjtRQUNILENBQUM7S0FBQTtJQUVEOztPQUVHO0lBQ0ssTUFBTTtRQUNaLElBQUksSUFBSSxDQUFDLFFBQVEsS0FBSyxJQUFJLEVBQUU7WUFDMUIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDLENBQUM7U0FDN0I7SUFDSCxDQUFDO0NBQ0Y7QUFFRCxrQkFBZSxxQkFBcUIsQ0FBQyJ9
\ No newline at end of file
dist/analyser-transformable.js 0 → 100644
View file @ 918e8c44
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const ramda = require("ramda");
const stream_1 = require("stream");
/**
* Decode a stream of a json array.
* emit single json objects
*/
class AnalyserTransformable extends stream_1.Transform {
/**
* @inheritDoc
* @param {{}} options
* @param uriBuilder UrlBuilder
* @param nconf
*/
constructor(options = {}, uriBuilder, nconf) {
super(options);
/**
* PropertyCollection to extract from wiki data structure from lena
* todo: get this dynamic
* @type {string[]}
*/
this.normIdentifierProperties = [
'P1005',
'P1017',
'P1225',
'P1255',
'P1281',
'P1417',
'P1566',
'P1667',
'P1937',
'P213',
'P214',
'P2163',
'P227',
'P244',
'P2581',
'P2612',
'P268',
'P269',
'P2924',
'P297',
'P298',
'P299',
'P3219',
'P3241',
'P3417',
'P3500',
'P3961',
'P402',
'P409',
'P4212',
'P4427',
'P4672',
'P5019',
'P605',
'P646',
'P648',
'P771',
'P901',
'P902',
'P906',
'P949',
'P950',
'P984',
'P998'
];
this.urlBuilder = uriBuilder;
this.nconf = nconf;
}
/**
* @inheritDoc
* @param chunk
* @param encoding
* @param callback
* @private
*/
_transform(chunk, encoding, callback) {
// if norm_identifiers are unset, go ahead
const data = Buffer.from(chunk.parsed.data).toString('utf8');
const obj = JSON.parse(data);
if (typeof obj.claims === 'undefined') {
return callback(null);
}
// check if there are norm_identifier
const to = this.normIdentifier(obj);
if (to.length === 0) {
return callback(null);
}
// prepare things
const result = {
from: this.urlBuilder.urlResolver('wikidata', obj.id),
to,
relation: {
type: 'same_as',
author: this.nconf.get('producer:analyser:author'),
weight: 1,
},
};
callback(null, result);
}
/**
* Get a list of object to describe norm data
* @param obj
*/
normIdentifier(obj) {
return Object.keys(obj.claims)
.filter((key) => {
return this.normIdentifierProperties.indexOf(key) > -1;
})
.map((key) => {
return this.urlBuilder.urlResolver(key, ramda.path(['claims', key, 0, 'mainsnak', 'datavalue', 'value'], obj));
})
.filter((nValue) => nValue);
}
}
exports.default = AnalyserTransformable;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYW5hbHlzZXItdHJhbnNmb3JtYWJsZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9hbmFseXNlci10cmFuc2Zvcm1hYmxlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQ0EsK0JBQStCO0FBQy9CLG1DQUFpQztBQWVqQzs7O0dBR0c7QUFDSCwyQkFBNEIsU0FBUSxrQkFBUztJQWdFM0M7Ozs7O09BS0c7SUFDSCxZQUFZLE9BQU8sR0FBRyxFQUFFLEVBQUUsVUFBc0IsRUFBRSxLQUFLO1FBQ3JELEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQztRQXJFakI7Ozs7V0FJRztRQUNLLDZCQUF3QixHQUFhO1lBQzNDLE9BQU87WUFDUCxPQUFPO1lBQ1AsT0FBTztZQUNQLE9BQU87WUFDUCxPQUFPO1lBQ1AsT0FBTztZQUNQLE9BQU87WUFDUCxPQUFPO1lBQ1AsT0FBTztZQUNQLE1BQU07WUFDTixNQUFNO1lBQ04sT0FBTztZQUNQLE1BQU07WUFDTixNQUFNO1lBQ04sT0FBTztZQUNQLE9BQU87WUFDUCxNQUFNO1lBQ04sTUFBTTtZQUNOLE9BQU87WUFDUCxNQUFNO1lBQ04sTUFBTTtZQUNOLE1BQU07WUFDTixPQUFPO1lBQ1AsT0FBTztZQUNQLE9BQU87WUFDUCxPQUFPO1lBQ1AsT0FBTztZQUNQLE1BQU07WUFDTixNQUFNO1lBQ04sT0FBTztZQUNQLE9BQU87WUFDUCxPQUFPO1lBQ1AsT0FBTztZQUNQLE1BQU07WUFDTixNQUFNO1lBQ04sTUFBTTtZQUNOLE1BQU07WUFDTixNQUFNO1lBQ04sTUFBTTtZQUNOLE1BQU07WUFDTixNQUFNO1lBQ04sTUFBTTtZQUNOLE1BQU07WUFDTixNQUFNO1NBQUMsQ0FBQztRQXFCUixJQUFJLENBQUMsVUFBVSxHQUFHLFVBQVUsQ0FBQztRQUM3QixJQUFJLENBQUMsS0FBSyxHQUFHLEtBQUssQ0FBQztJQUNyQixDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ0ksVUFBVSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUTtRQUN6QywwQ0FBMEM7UUFDMUMsTUFBTSxJQUFJLEdBQUcsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUM3RCxNQUFNLEdBQUcsR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDO1FBRTdCLElBQUksT0FBTyxHQUFHLENBQUMsTUFBTSxLQUFLLFdBQVcsRUFBRTtZQUNyQyxPQUFPLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQztTQUN2QjtRQUVELHFDQUFxQztRQUNyQyxNQUFNLEVBQUUsR0FBRyxJQUFJLENBQUMsY0FBYyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1FBQ3BDLElBQUksRUFBRSxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUc7WUFDcEIsT0FBTyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUM7U0FDdkI7UUFFRCxpQkFBaUI7UUFDakIsTUFBTSxNQUFNLEdBQWtCO1lBQzVCLElBQUksRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLFdBQVcsQ0FBQyxVQUFVLEVBQUUsR0FBRyxDQUFDLEVBQUUsQ0FBQztZQUNyRCxFQUFFO1lBQ0YsUUFBUSxFQUFFO2dCQUNSLElBQUksRUFBRSxTQUFTO2dCQUNmLE1BQU0sRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQywwQkFBMEIsQ0FBQztnQkFDbEQsTUFBTSxFQUFFLENBQUM7YUFDVjtTQUNGLENBQUM7UUFDRixRQUFRLENBQUMsSUFBSSxFQUFFLE1BQU0sQ0FBQyxDQUFDO0lBQ3pCLENBQUM7SUFFRDs7O09BR0c7SUFDSyxjQUFjLENBQUMsR0FBRztRQUN4QixPQUFPLE1BQU0sQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQzthQUMzQixNQUFNLENBQUMsQ0FBQyxHQUFHLEVBQUUsRUFBRTtZQUNkLE9BQU8sSUFBSSxDQUFDLHdCQUF3QixDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztRQUN6RCxDQUFDLENBQUM7YUFDRCxHQUFHLENBQUMsQ0FBQyxHQUFXLEVBQUUsRUFBRTtZQUNuQixPQUFPLElBQUksQ0FBQyxVQUFVLENBQUMsV0FBVyxDQUFDLEdBQUcsRUFBRSxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUMsUUFBUSxFQUFFLEdBQUcsRUFBRSxDQUFDLEVBQUUsVUFBVSxFQUFFLFdBQVcsRUFBRSxPQUFPLENBQUMsRUFBRSxHQUFHLENBQUMsQ0FBQyxDQUFDO1FBQ2pILENBQUMsQ0FBQzthQUNELE1BQU0sQ0FBQyxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDaEMsQ0FBQztDQWFGO0FBQ0Qsa0JBQWUscUJBQXFCLENBQUMifQ==
dist/bootstrap-wikidata.js 0 → 100644
View file @ 918e8c44
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const wikidata_taxonomy_1 = require("wikidata-taxonomy");
class BootstrapWikidata {
constructor() {
/**
* An array of properties to load from wikidata
* @type {any[]}
*/
this.properties = [];
}
/**
* Load a list of properties with subclasses from wikidata
* @param {InterfaceProperty[]} properties
* @returns {Promise<void>}
*/
init(properties) {
return __awaiter(this, void 0, void 0, function* () {
for (const property of properties) {
const sublclasses = yield this.getProperties(property.qid);
// add the main property to the list
sublclasses.push(property.qid);
this.properties.push({ name: property.name, properties: sublclasses });
}
});
}
/**
* Get a collection of subclasses for a named property
* @param {string} name
* @returns {string[]}
*/
get(name) {
const property = this.properties.find((p) => {
return p.name === name;
});
return property.properties;
}
/**
* Get a property with his subclasses from wikidata
* @param {string} property
* @returns {Promise<any>}
*/
getProperties(property) {
return __awaiter(this, void 0, void 0, function* () {
// P1647 = subproperty
return yield wikidata_taxonomy_1.queryTaxonomy(property, { brief: true, property: ['P1647'] })
.then((taxonomy) => {
const properties = [];
taxonomy.concepts.forEach((concept) => {
const qid = concept.notation[0];
properties.push(qid);
});
return properties.filter((d, i, a) => {
return a.indexOf(d) === i;
});
})
.catch((error) => {
console.log('error while collectin Qids');
console.log(error);
});
});
}
}
exports.BootstrapWikidata = BootstrapWikidata;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYm9vdHN0cmFwLXdpa2lkYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2Jvb3RzdHJhcC13aWtpZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7O0FBQUEseURBQWtEO0FBT2xEO0lBQUE7UUFDRTs7O1dBR0c7UUFDSyxlQUFVLEdBQUcsRUFBRSxDQUFDO0lBbUQxQixDQUFDO0lBakRDOzs7O09BSUc7SUFDVSxJQUFJLENBQUMsVUFBK0I7O1lBQy9DLEtBQUssTUFBTSxRQUFRLElBQUksVUFBVSxFQUFFO2dCQUMvQixNQUFNLFdBQVcsR0FBRyxNQUFNLElBQUksQ0FBQyxhQUFhLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDO2dCQUMzRCxvQ0FBb0M7Z0JBQ3BDLFdBQVcsQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDO2dCQUMvQixJQUFJLENBQUMsVUFBVSxDQUFDLElBQUksQ0FBQyxFQUFDLElBQUksRUFBRSxRQUFRLENBQUMsSUFBSSxFQUFFLFVBQVUsRUFBRSxXQUFXLEVBQUMsQ0FBQyxDQUFDO2FBQ3RFO1FBQ0wsQ0FBQztLQUFBO0lBRUQ7Ozs7T0FJRztJQUNJLEdBQUcsQ0FBQyxJQUFZO1FBQ3JCLE1BQU0sUUFBUSxHQUFHLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUU7WUFDMUMsT0FBTyxDQUFDLENBQUMsSUFBSSxLQUFLLElBQUksQ0FBQztRQUN6QixDQUFDLENBQUMsQ0FBQztRQUNILE9BQU8sUUFBUSxDQUFDLFVBQVUsQ0FBQztJQUM3QixDQUFDO0lBRUQ7Ozs7T0FJRztJQUNXLGFBQWEsQ0FBQyxRQUFnQjs7WUFDMUMsc0JBQXNCO1lBQ3RCLE9BQU8sTUFBTSxpQ0FBYSxDQUFDLFFBQVEsRUFBRSxFQUFDLEtBQUssRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLENBQUMsT0FBTyxDQUFDLEVBQUMsQ0FBQztpQkFDckUsSUFBSSxDQUFDLENBQUMsUUFBUSxFQUFFLEVBQUU7Z0JBQ2pCLE1BQU0sVUFBVSxHQUFHLEVBQUUsQ0FBQztnQkFDdEIsUUFBUSxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsQ0FBQyxPQUFPLEVBQUUsRUFBRTtvQkFDcEMsTUFBTSxHQUFHLEdBQUcsT0FBTyxDQUFDLFFBQVEsQ0FBQyxDQUFDLENBQUMsQ0FBQztvQkFDaEMsVUFBVSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztnQkFDdkIsQ0FBQyxDQUFDLENBQUM7Z0JBQ0gsT0FBTyxVQUFVLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRTtvQkFDbkMsT0FBTyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQztnQkFDNUIsQ0FBQyxDQUFDLENBQUM7WUFDTCxDQUFDLENBQUM7aUJBQ0QsS0FBSyxDQUFDLENBQUMsS0FBSyxFQUFFLEVBQUU7Z0JBQ2YsT0FBTyxDQUFDLEdBQUcsQ0FBQyw0QkFBNEIsQ0FBQyxDQUFDO2dCQUMxQyxPQUFPLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDO1lBQ3JCLENBQUMsQ0FBQyxDQUFDO1FBQ1AsQ0FBQztLQUFBO0NBQ0Y7QUF4REQsOENBd0RDIn0=
\ No newline at end of file
dist/debug.js 0 → 100644
View file @ 918e8c44
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
class Debug extends stream_1.Transform {
constructor(opts, text) {
super(opts);
this.text = text;
}
/**
* @inheritDoc
* @param chunk
* @param {string} encoding
* @param {(err?: Error) => void} callback
* @private
*/
_transform(chunk, encoding, callback) {
callback(null, chunk);
console.log(this.text);
console.log(chunk);
}
}
exports.default = Debug;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZGVidWcuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvZGVidWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7QUFBQSxtQ0FBaUM7QUFFakMsV0FBWSxTQUFRLGtCQUFTO0lBRzNCLFlBQVksSUFBSSxFQUFFLElBQVk7UUFDNUIsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDO1FBQ1osSUFBSSxDQUFDLElBQUksR0FBRyxJQUFJLENBQUM7SUFDbkIsQ0FBQztJQUNEOzs7Ozs7T0FNRztJQUNJLFVBQVUsQ0FBQyxLQUFLLEVBQUUsUUFBUSxFQUFFLFFBQVE7UUFDekMsUUFBUSxDQUFDLElBQUksRUFBRSxLQUFLLENBQUMsQ0FBQztRQUN0QixPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUN2QixPQUFPLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3JCLENBQUM7Q0FDRjtBQUVELGtCQUFlLEtBQUssQ0FBQyJ9
\ No newline at end of file
dist/geolinker-stream-preparer.js 0 → 100644
View file @ 918e8c44
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
/**
* Prepare a value for kafka ProducerStream
*/
class StreamProducerPreparer extends stream_1.Transform {
constructor(opts) {
super(opts);
/**
* Partition
*/
this.partition = -1;
this.topic = opts.topic;
this.partition = opts.partition || -1;
}
/**
* @inheritDoc
* @param chunk
* @param {string} encoding
* @param {(err?: Error) => void} callback
* @private
*/
_transform(chunk, encoding, callback) {
// see: producer-stream.js L: 176
const value = {
topic: this.topic,
value: chunk,
partition: this.partition,
};
callback(null, value);
}
}
exports.default = StreamProducerPreparer;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2VvbGlua2VyLXN0cmVhbS1wcmVwYXJlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9nZW9saW5rZXItc3RyZWFtLXByZXBhcmVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQUEsbUNBQW1DO0FBRW5DOztHQUVHO0FBQ0gsNEJBQTZCLFNBQVEsa0JBQVM7SUFZNUMsWUFBWSxJQUFJO1FBQ2QsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDO1FBTmQ7O1dBRUc7UUFDYyxjQUFTLEdBQVcsQ0FBQyxDQUFDLENBQUM7UUFJdEMsSUFBSSxDQUFDLEtBQUssR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDO1FBQ3hCLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLFNBQVMsSUFBSSxDQUFDLENBQUMsQ0FBQztJQUN4QyxDQUFDO0lBQ0Q7Ozs7OztPQU1HO0lBQ0ksVUFBVSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUTtRQUN6QyxpQ0FBaUM7UUFDakMsTUFBTSxLQUFLLEdBQUc7WUFDWixLQUFLLEVBQUUsSUFBSSxDQUFDLEtBQUs7WUFDakIsS0FBSyxFQUFFLEtBQUs7WUFDWixTQUFTLEVBQUUsSUFBSSxDQUFDLFNBQVM7U0FDMUIsQ0FBQztRQUNGLFFBQVEsQ0FBQyxJQUFJLEVBQUUsS0FBSyxDQUFDLENBQUM7SUFDeEIsQ0FBQztDQUNGO0FBRUQsa0JBQWUsc0JBQXNCLENBQUMifQ==
\ No newline at end of file
dist/geolinker-stream-producer.js 0 → 100644
View file @ 918e8c44
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
/**
* Prepare a value for kafka ProducerStream
*/
class StreamProducerPreparer extends stream_1.Transform {
constructor(opts) {
super(opts);
/**
* Partition
*/
this.partition = -1;
this.topic = opts.topic;
this.partition = opts.partition || -1;
}
/**
* @inheritDoc
* @param chunk
* @param {string} encoding
* @param {(err?: Error) => void} callback
* @private
*/
_transform(chunk, encoding, callback) {
// see: producer-stream.js L: 176
const value = {
topic: this.topic,
value: chunk,
partition: this.partition,
};
callback(null, value);
}
}
exports.default = StreamProducerPreparer;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2VvbGlua2VyLXN0cmVhbS1wcm9kdWNlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9nZW9saW5rZXItc3RyZWFtLXByb2R1Y2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQUEsbUNBQW1DO0FBRW5DOztHQUVHO0FBQ0gsNEJBQTZCLFNBQVEsa0JBQVM7SUFZNUMsWUFBWSxJQUFJO1FBQ2QsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDO1FBTmQ7O1dBRUc7UUFDYyxjQUFTLEdBQVcsQ0FBQyxDQUFDLENBQUM7UUFJdEMsSUFBSSxDQUFDLEtBQUssR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDO1FBQ3hCLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLFNBQVMsSUFBSSxDQUFDLENBQUMsQ0FBQztJQUN4QyxDQUFDO0lBQ0Q7Ozs7OztPQU1HO0lBQ0ksVUFBVSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUTtRQUN6QyxpQ0FBaUM7UUFDakMsTUFBTSxLQUFLLEdBQUc7WUFDWixLQUFLLEVBQUUsSUFBSSxDQUFDLEtBQUs7WUFDakIsS0FBSyxFQUFFLEtBQUs7WUFDWixTQUFTLEVBQUUsSUFBSSxDQUFDLFNBQVM7U0FDMUIsQ0FBQztRQUNGLFFBQVEsQ0FBQyxJQUFJLEVBQUUsS0FBSyxDQUFDLENBQUM7SUFDeEIsQ0FBQztDQUNGO0FBRUQsa0JBQWUsc0JBQXNCLENBQUMifQ==
\ No newline at end of file
dist/index.js 0 → 100644
View file @ 918e8c44
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const fmt = require("bunyan-format");
const geolinker_common_1 = require("geolinker-common");
const reporter_counter_1 = require("geolinker-common/dist/stream/reporter-counter");
const stream_producer_preparer_1 = require("geolinker-common/dist/stream/stream-producer-preparer");
const KafkaAvro = require("kafka-avro");
const nodeCleanup = require("node-cleanup");
const ProducerStream = require("node-rdkafka/lib/producer-stream");
const analyser_transformable_1 = require("./analyser-transformable");
const bootstrap_wikidata_1 = require("./bootstrap-wikidata");
const wiki_normalizer_1 = require("./wiki-normalizer");
const debug_1 = require("geolinker-common/dist/stream/debug");
/**
* load sourcemap and config
*/
const core = geolinker_common_1.Core.init('wikidata-normalizer', __dirname + '/../config.json');
console.log("1");
/**
* To trigger this you need to set the KAFKA_AVRO_LOG_LEVEL
* f.e. env KAFKA_AVRO_LOG_LEVEL=debug node dist/index.js
*/
const kafkaLog = KafkaAvro.getLogger();
kafkaLog.addStream({
type: 'stream',
stream: fmt({
outputMode: 'short',
levelInString: true,
}),
level: 'debug',
});
nodeCleanup((exitCode, signal) => {
core.getLogger().info('Signal ' + signal);
core.getLogger().info('ExitCode ' + exitCode);
setTimeout(() => {
process.exit();
}, 100);
});
/**
* A list of properties to load from wikidata
* @type {InterfaceProperty[]}
*/
const properties = [
{ name: 'start', qid: 'P580' },
{ name: 'end', qid: 'P582' },
];
const wikidataProperties = new bootstrap_wikidata_1.BootstrapWikidata();
const wpPromise = wikidataProperties.init(properties);
/**
* Init KafkaInstance
* @type {}
*/
const kafka = core.getKafkaAvro();
kafka.init().then(() => __awaiter(this, void 0, void 0, function* () {
console.log("beta");
// Wait for the promises to resolve
try {
yield wpPromise;
const analyseProducer = yield kafka.getProducer('analyser');
const normProducer = yield kafka.getProducer('normalizer');
const consumerStream = yield kafka.getConsumerStream();
core.getLogger().info('Producer and consumer are ready to go');
/**
* On data (flow mode) transform data and send them to the data topic
*/
consumerStream
.pipe(new reporter_counter_1.default({ objectMode: true }, core.getReporter(10000, 'normalizer'), 'read'))
.pipe(new wiki_normalizer_1.default({ objectMode: true }, core.getLogger(), wikidataProperties))
.pipe(new reporter_counter_1.default({ objectMode: true }, core.getReporter('normalizer'), 'write'))
.pipe(new stream_producer_preparer_1.default({ objectMode: true, topic: core.getNconf().get('producer:normalizer:topics'), partition: -1 }))
.pipe(new debug_1.default({ objectMode: true }, '---'))
.pipe(new ProducerStream(normProducer, { objectMode: true, topic: core.getNconf().get('producer:normalizer:topics') }));
// todo: we need a second consumer and we need a second reporter. So we need to update common libs
consumerStream
.pipe(new reporter_counter_1.default({ objectMode: true }, core.getReporter(10000, 'analyser'), 'read'))
.pipe(new analyser_transformable_1.default({ objectMode: true }, core.getUriBuilder(), core.getNconf()))
.pipe(new reporter_counter_1.default({ objectMode: true }, core.getReporter(10000, 'analyser'), 'write'))
.pipe(new stream_producer_preparer_1.default({ objectMode: true, topic: core.getNconf().get('producer:analyser:topics'), partition: -1, key: (d) => `${d.provider}${d.id}` }))
.pipe(new ProducerStream(analyseProducer, { objectMode: true, topic: core.getNconf().get('producer:analyser:topics') }));
core.getLogger().info('Pipes are registered');
}
catch (error) {
core.getLogger().error('Error in retrieving consumer or producer');
core.getLogger().error(error);
}
}))
.catch((err) => {
core.getLogger().error('Its not possible to connect to kafka');
throw err;
});
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7OztBQUFBLHFDQUFzQztBQUN0Qyx1REFBc0M7QUFDdEMsb0ZBQTRFO0FBQzVFLG9HQUEyRjtBQUMzRix3Q0FBd0M7QUFDeEMsNENBQTZDO0FBQzdDLG1FQUFvRTtBQUNwRSxxRUFBNkQ7QUFDN0QsNkRBQTBFO0FBQzFFLHVEQUErQztBQUMvQyw4REFBdUQ7QUFFdkQ7O0dBRUc7QUFDSCxNQUFNLElBQUksR0FBRyx1QkFBSSxDQUFDLElBQUksQ0FBQyxxQkFBcUIsRUFBRSxTQUFTLEdBQUcsaUJBQWlCLENBQUMsQ0FBQztBQUM3RSxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0FBQ2pCOzs7R0FHRztBQUNILE1BQU0sUUFBUSxHQUFHLFNBQVMsQ0FBQyxTQUFTLEVBQUUsQ0FBQztBQUV2QyxRQUFRLENBQUMsU0FBUyxDQUFDO0lBQ2pCLElBQUksRUFBRSxRQUFRO0lBQ2QsTUFBTSxFQUFFLEdBQUcsQ0FBQztRQUNWLFVBQVUsRUFBRSxPQUFPO1FBQ25CLGFBQWEsRUFBRSxJQUFJO0tBQ3BCLENBQUM7SUFDRixLQUFLLEVBQUUsT0FBTztDQUNmLENBQUMsQ0FBQztBQUVILFdBQVcsQ0FBQyxDQUFDLFFBQVEsRUFBRSxNQUFNLEVBQUUsRUFBRTtJQUMvQixJQUFJLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxDQUFDLFNBQVMsR0FBRyxNQUFNLENBQUMsQ0FBQztJQUMxQyxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxDQUFDLFdBQVcsR0FBRyxRQUFRLENBQUMsQ0FBQztJQUM5QyxVQUFVLENBQUMsR0FBRyxFQUFFO1FBQ2QsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ2pCLENBQUMsRUFBRSxHQUFHLENBQUMsQ0FBQztBQUNWLENBQUMsQ0FBQyxDQUFDO0FBRUg7OztHQUdHO0FBQ0gsTUFBTSxVQUFVLEdBQUc7SUFDakIsRUFBQyxJQUFJLEVBQUUsT0FBTyxFQUFFLEdBQUcsRUFBRSxNQUFNLEVBQUM7SUFDNUIsRUFBQyxJQUFJLEVBQUUsS0FBSyxFQUFFLEdBQUcsRUFBRSxNQUFNLEVBQUM7Q0FDM0IsQ0FBQztBQUNGLE1BQU0sa0JBQWtCLEdBQUcsSUFBSSxzQ0FBaUIsRUFBRSxDQUFDO0FBQ25ELE1BQU0sU0FBUyxHQUFHLGtCQUFrQixDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztBQUV0RDs7O0dBR0c7QUFDSCxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLENBQUM7QUFDbEMsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDLElBQUksQ0FBQyxHQUFTLEVBQUU7SUFDM0IsT0FBTyxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUNwQixtQ0FBbUM7SUFDbkMsSUFBSTtRQUNGLE1BQU0sU0FBUyxDQUFDO1FBQ2hCLE1BQU0sZUFBZSxHQUFHLE1BQU0sS0FBSyxDQUFDLFdBQVcsQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUM1RCxNQUFNLFlBQVksR0FBRyxNQUFNLEtBQUssQ0FBQyxXQUFXLENBQUMsWUFBWSxDQUFDLENBQUM7UUFDM0QsTUFBTSxjQUFjLEdBQUcsTUFBTSxLQUFLLENBQUMsaUJBQWlCLEVBQUUsQ0FBQztRQUV2RCxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxDQUFDLHVDQUF1QyxDQUFDLENBQUM7UUFFL0Q7O1dBRUc7UUFDSCxjQUFjO2FBQ1gsSUFBSSxDQUFDLElBQUksMEJBQWUsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxJQUFJLENBQUMsV0FBVyxDQUFDLEtBQUssRUFBRSxZQUFZLENBQUMsRUFBRSxNQUFNLENBQUMsQ0FBQzthQUM1RixJQUFJLENBQUMsSUFBSSx5QkFBYyxDQUFDLEVBQUMsVUFBVSxFQUFFLElBQUksRUFBQyxFQUFFLElBQUksQ0FBQyxTQUFTLEVBQUUsRUFBRSxrQkFBa0IsQ0FBQyxDQUFDO2FBQ2xGLElBQUksQ0FBQyxJQUFJLDBCQUFlLENBQUMsRUFBQyxVQUFVLEVBQUUsSUFBSSxFQUFDLEVBQUUsSUFBSSxDQUFDLFdBQVcsQ0FBQyxZQUFZLENBQUMsRUFBRSxPQUFPLENBQUMsQ0FBQzthQUN0RixJQUFJLENBQUMsSUFBSSxrQ0FBc0IsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUUsS0FBSyxFQUFFLElBQUksQ0FBQyxRQUFRLEVBQUUsQ0FBQyxHQUFHLENBQUMsNEJBQTRCLENBQUMsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDLEVBQUMsQ0FBQyxDQUFDO2FBQzdILElBQUksQ0FBQyxJQUFJLGVBQUssQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxLQUFLLENBQUMsQ0FBQzthQUMxQyxJQUFJLENBQUMsSUFBSSxjQUFjLENBQUMsWUFBWSxFQUFFLEVBQUMsVUFBVSxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLEdBQUcsQ0FBQyw0QkFBNEIsQ0FBQyxFQUFDLENBQUMsQ0FBQyxDQUFDO1FBRXhILGtHQUFrRztRQUNsRyxjQUFjO2FBQ1gsSUFBSSxDQUFDLElBQUksMEJBQWUsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxJQUFJLENBQUMsV0FBVyxDQUFDLEtBQUssRUFBRSxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUMsQ0FBQzthQUMxRixJQUFJLENBQUMsSUFBSSxnQ0FBcUIsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxJQUFJLENBQUMsYUFBYSxFQUFFLEVBQUUsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7YUFDMUYsSUFBSSxDQUFDLElBQUksMEJBQWUsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxJQUFJLENBQUMsV0FBVyxDQUFDLEtBQUssRUFBRSxVQUFVLENBQUMsRUFBRSxPQUFPLENBQUMsQ0FBQzthQUMzRixJQUFJLENBQUMsSUFBSSxrQ0FBc0IsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUUsS0FBSyxFQUFFLElBQUksQ0FBQyxRQUFRLEVBQUUsQ0FBQyxHQUFHLENBQUMsMEJBQTBCLENBQUMsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDLEVBQUUsR0FBRyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxHQUFHLENBQUMsQ0FBQyxRQUFRLEdBQUcsQ0FBQyxDQUFDLEVBQUUsRUFBRSxFQUFDLENBQUMsQ0FBQzthQUMvSixJQUFJLENBQUMsSUFBSSxjQUFjLENBQUMsZUFBZSxFQUFFLEVBQUMsVUFBVSxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLEdBQUcsQ0FBQywwQkFBMEIsQ0FBQyxFQUFDLENBQUMsQ0FBQyxDQUFDO1FBQ3pILElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLENBQUMsc0JBQXNCLENBQUMsQ0FBQztLQUMvQztJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ2QsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDLEtBQUssQ0FBQywwQ0FBMEMsQ0FBQyxDQUFDO1FBQ25FLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLENBQUM7S0FDL0I7QUFFSCxDQUFDLENBQUEsQ0FBQztLQUNDLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO0lBQ2IsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDLEtBQUssQ0FBQyxzQ0FBc0MsQ0FBQyxDQUFDO0lBQy9ELE1BQU0sR0FBRyxDQUFDO0FBQ1osQ0FBQyxDQUFDLENBQUMifQ==
\ No newline at end of file
dist/kafka.js 0 → 100644
View file @ 918e8c44
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const KafkaAvro = require("kafka-avro");
const nconf = require("nconf");
const logger_1 = require("geolinker-common/dist/logger");
/**
* Load config for the streamer
*/
nconf.argv()
.env()
.file({ file: './config.json' });
const kafkaAvroInstance = new KafkaAvro({
kafkaBroker: nconf.get('kafka:broker'),
schemaRegistry: nconf.get('kafka:schema-registry'),
});
// Wait until kafka is initialised then return an instance
exports.default = () => __awaiter(this, void 0, void 0, function* () {
return yield kafkaAvroInstance.init()
.then(() => {
logger_1.default.info('Ready to consume');
return kafkaAvroInstance;
});
});
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoia2Fma2EuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMva2Fma2EudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7OztBQUFBLHdDQUF5QztBQUN6QywrQkFBZ0M7QUFDaEMseURBQWtEO0FBRWxEOztHQUVHO0FBQ0gsS0FBSyxDQUFDLElBQUksRUFBRTtLQUNULEdBQUcsRUFBRTtLQUNMLElBQUksQ0FBQyxFQUFDLElBQUksRUFBRSxlQUFlLEVBQUMsQ0FBQyxDQUFDO0FBRWpDLE1BQU0saUJBQWlCLEdBQUcsSUFBSSxTQUFTLENBQUM7SUFDdEMsV0FBVyxFQUFFLEtBQUssQ0FBQyxHQUFHLENBQUMsY0FBYyxDQUFDO0lBQ3RDLGNBQWMsRUFBRSxLQUFLLENBQUMsR0FBRyxDQUFDLHVCQUF1QixDQUFDO0NBQ25ELENBQUMsQ0FBQztBQUVILDBEQUEwRDtBQUMxRCxrQkFBZSxHQUFTLEVBQUU7SUFDeEIsT0FBTyxNQUFNLGlCQUFpQixDQUFDLElBQUksRUFBRTtTQUNsQyxJQUFJLENBQUMsR0FBRyxFQUFFO1FBQ1QsZ0JBQU0sQ0FBQyxJQUFJLENBQUMsa0JBQWtCLENBQUMsQ0FBQztRQUNoQyxPQUFPLGlCQUFpQixDQUFDO0lBQzNCLENBQUMsQ0FBQyxDQUFDO0FBQ1AsQ0FBQyxDQUFBLENBQUMifQ==
\ No newline at end of file
dist/utils/property-collection.js 0 → 100644
View file @ 918e8c44
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
/**
* Simple class to manage a collection of properties
*/
class PropertyCollection {
constructor() {
/**
* An array of properties to load from wikidata
* @type {any[]}
*/
this.properties = [];
}
/**
* Get a collection of subclasses for a named property
* @param {string} name
* @returns {string[]}
*/
get(name) {
const property = this.properties.find((p) => {
return p.name === name;
});
return property.subclasses;
}
/**
* Add a collection of subclasses for a named propertiy
* @param {string} name
* @param {string[]} subclasses
* @returns {void}
*/
add(name, subclasses) {
this.properties.push({ name, subclasses });
}
}
exports.PropertyCollection = PropertyCollection;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvcGVydHktY29sbGVjdGlvbi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy91dGlscy9wcm9wZXJ0eS1jb2xsZWN0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBSUE7O0dBRUc7QUFDSCxNQUFhLGtCQUFrQjtJQUEvQjtRQUNFOzs7V0FHRztRQUNLLGVBQVUsR0FBRyxFQUFFLENBQUM7SUF1QjFCLENBQUM7SUFyQkM7Ozs7T0FJRztJQUNJLEdBQUcsQ0FBQyxJQUFZO1FBQ3JCLE1BQU0sUUFBUSxHQUFHLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUU7WUFDMUMsT0FBTyxDQUFDLENBQUMsSUFBSSxLQUFLLElBQUksQ0FBQztRQUN6QixDQUFDLENBQUMsQ0FBQztRQUNILE9BQU8sUUFBUSxDQUFDLFVBQVUsQ0FBQztJQUM3QixDQUFDO0lBRUQ7Ozs7O09BS0c7SUFDSSxHQUFHLENBQUMsSUFBWSxFQUFFLFVBQW9CO1FBQzNDLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLEVBQUMsSUFBSSxFQUFFLFVBQVUsRUFBQyxDQUFDLENBQUM7SUFDM0MsQ0FBQztDQUNGO0FBNUJELGdEQTRCQyJ9
\ No newline at end of file
dist/utils/wikidata-properties.js 0 → 100644
View file @ 918e8c44
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const wikidata_taxonomy_1 = require("wikidata-taxonomy");
const property_collection_1 = require("./property-collection");
class WikidataProperties {
/**
* Get a property with his subclasses from wikidata
* @param {string} property
* @param {{}} query f.e. P1647 = subproperty
* @returns {Promise<any>}
*/
static getProperties(property, query = { brief: true }) {
return __awaiter(this, void 0, void 0, function* () {
try {
const taxonomy = yield wikidata_taxonomy_1.queryTaxonomy(property, query);
const props = [];
taxonomy.concepts.forEach((concept) => {
const qid = concept.notation[0];
props.push(qid);
});
return props.filter((d, i, a) => {
return a.indexOf(d) === i;
});
}
catch (error) {
console.log('Error while collecting Q-numbers from wikidata');
console.log(error);
return [];
}
});
}
/**
* Load a list of properties with subclasses from wikidata
* @param {IProperty[]} props
* @returns {Promise<PropertyCollection>}
*/
static init(props) {
return __awaiter(this, void 0, void 0, function* () {
const properties = new property_collection_1.PropertyCollection();
for (const property of props) {
const sublclasses = yield WikidataProperties.getProperties(property.qid, { brief: true, property: ['P1647'] });
// add the main property to the list
sublclasses.push(property.qid);
properties.add(property.name, sublclasses);
}