Commit eb015d5d authored by Tobinsk's avatar Tobinsk
Browse files

Resolve "Turn into lib"

parent 9887f5fa
......@@ -59,5 +59,4 @@ typings/
# next.js build output
.next
dist/
.idea/
\ No newline at end of file
.idea/
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
const Spider = require("node-spider");
/**
* Verbosity
* @type {boolean}
*/
const debug = false;
/**
* Small loger for the spider
* @type {{write: function(*=)}}
*/
const logger = {
write: (msg) => {
if (debug) {
console.log(msg);
}
}
};
/**
* A small writer to put result to stdout
* @type {{put: function(*=)}}
*/
const writer = {
put: (line) => {
console.log(line);
}
};
const spider = new Spider({
// How many requests can be run in parallel
concurrent: 20,
// How long to wait after each request
delay: 50,
// A stream to where internal logs are sent, optional
logs: logger,
// Re-visit visited URLs, false by default
allowDuplicates: false,
// If `true` all queued handlers will be try-catch'd, errors go to `error` callback
catchErrors: false,
// If `true` the spider will set the Referer header automatically on subsequent requests
addReferrer: false,
// If `true` adds the X-Requested-With:XMLHttpRequest header
xhr: false,
// If `true` adds the Connection:keep-alive header and forever option on request module
keepAlive: false,
// Called when there's an error, throw will be used if none is provided
error: function (err, url) {
console.log(`There was an error while fetching ${url}` + err);
},
// Called when there are no more requests
done: function () {
console.log("We are done");
},
headers: { 'user-agent': 'magpie' },
encoding: 'utf8'
});
class Crawler extends stream_1.Transform {
constructor(options, logger) {
super(options);
/**
* Timeout to queue an url if the queue is full
* @type {number}
*/
this.timeout = 20;
this.logger = logger;
}
/**
* @inheritDoc
* @param chunk
* @param encoding
* @param callback
* @private
*/
_transform(chunk, encoding, callback) {
console.log('loaded');
if (!spider.full()) {
const data = chunk.parsed;
spider.queue(data.url, (doc) => {
console.log("Successfully got data " + data.url);
data.doc = doc;
callback(null, data);
}, data.headers);
}
else {
setTimeout(() => {
this._transform(chunk, encoding, callback);
}, this.timeout);
}
}
}
exports.default = Crawler;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3Jhd2xlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9jcmF3bGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQUEsbUNBQWlDO0FBRWpDLHNDQUF1QztBQUV2Qzs7O0dBR0c7QUFDSCxNQUFNLEtBQUssR0FBRyxLQUFLLENBQUM7QUFFcEI7OztHQUdHO0FBQ0gsTUFBTSxNQUFNLEdBQUc7SUFDWCxLQUFLLEVBQUUsQ0FBQyxHQUFHLEVBQUUsRUFBRTtRQUNYLElBQUcsS0FBSyxFQUFFO1lBQ04sT0FBTyxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsQ0FBQztTQUNwQjtJQUNMLENBQUM7Q0FDSixDQUFDO0FBRUY7OztHQUdHO0FBQ0gsTUFBTSxNQUFNLEdBQUc7SUFDWCxHQUFHLEVBQUUsQ0FBQyxJQUFJLEVBQUUsRUFBRTtRQUNWLE9BQU8sQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDdEIsQ0FBQztDQUNKLENBQUM7QUFFRixNQUFNLE1BQU0sR0FBRyxJQUFJLE1BQU0sQ0FBQztJQUN0QiwyQ0FBMkM7SUFDM0MsVUFBVSxFQUFFLEVBQUU7SUFDZCxzQ0FBc0M7SUFDdEMsS0FBSyxFQUFFLEVBQUU7SUFDVCxxREFBcUQ7SUFDckQsSUFBSSxFQUFFLE1BQU07SUFDWiwwQ0FBMEM7SUFDMUMsZUFBZSxFQUFFLEtBQUs7SUFDdEIsbUZBQW1GO0lBQ25GLFdBQVcsRUFBRSxLQUFLO0lBQ2xCLHdGQUF3RjtJQUN4RixXQUFXLEVBQUUsS0FBSztJQUNsQiw0REFBNEQ7SUFDNUQsR0FBRyxFQUFFLEtBQUs7SUFDVix1RkFBdUY7SUFDdkYsU0FBUyxFQUFFLEtBQUs7SUFDaEIsdUVBQXVFO0lBQ3ZFLEtBQUssRUFBRSxVQUFTLEdBQUcsRUFBRSxHQUFHO1FBQ3BCLE9BQU8sQ0FBQyxHQUFHLENBQUMscUNBQXFDLEdBQUcsRUFBRSxHQUFHLEdBQUcsQ0FBQyxDQUFDO0lBQ2xFLENBQUM7SUFDRCx5Q0FBeUM7SUFDekMsSUFBSSxFQUFFO1FBQ0YsT0FBTyxDQUFDLEdBQUcsQ0FBQyxhQUFhLENBQUMsQ0FBQztJQUMvQixDQUFDO0lBQ0QsT0FBTyxFQUFFLEVBQUUsWUFBWSxFQUFFLFFBQVEsRUFBRTtJQUNuQyxRQUFRLEVBQUUsTUFBTTtDQUNuQixDQUFDLENBQUM7QUFJSCxhQUFjLFNBQVEsa0JBQVM7SUFZM0IsWUFBWSxPQUFPLEVBQUUsTUFBc0I7UUFDdkMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBWG5COzs7V0FHRztRQUNLLFlBQU8sR0FBRyxFQUFFLENBQUM7UUFRakIsSUFBSSxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUM7SUFDekIsQ0FBQztJQUVEOzs7Ozs7T0FNRztJQUNJLFVBQVUsQ0FBQyxLQUFLLEVBQUUsUUFBUSxFQUFFLFFBQVE7UUFDdkMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxRQUFRLENBQUMsQ0FBQztRQUN0QixJQUFHLENBQUMsTUFBTSxDQUFDLElBQUksRUFBRSxFQUFFO1lBQ2YsTUFBTSxJQUFJLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQztZQUMxQixNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQyxHQUFHLEVBQUUsRUFBRTtnQkFDM0IsT0FBTyxDQUFDLEdBQUcsQ0FBQyx3QkFBd0IsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQ2pELElBQUksQ0FBQyxHQUFHLEdBQUcsR0FBRyxDQUFDO2dCQUNmLFFBQVEsQ0FBQyxJQUFJLEVBQUUsSUFBSSxDQUFDLENBQUM7WUFDekIsQ0FBQyxFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztTQUNwQjthQUFNO1lBQ0gsVUFBVSxDQUFDLEdBQUcsRUFBRTtnQkFDWixJQUFJLENBQUMsVUFBVSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUSxDQUFDLENBQUM7WUFDL0MsQ0FBQyxFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztTQUNwQjtJQUNMLENBQUM7Q0FDSjtBQUVELGtCQUFlLE9BQU8sQ0FBQyJ9
\ No newline at end of file
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const geolinker_common_1 = require("geolinker-common");
const reporter_counter_1 = require("geolinker-common/dist/stream/reporter-counter");
const stream_producer_preparer_1 = require("geolinker-common/dist/stream/stream-producer-preparer");
const crawler_1 = require("./crawler");
const ProducerStream = require("node-rdkafka/lib/producer-stream");
const debug_1 = require("geolinker-common/dist/stream/debug");
/**
* load sourcemap and config
*/
const core = geolinker_common_1.Core.init('magpie', __dirname + '/../config.json');
/**
* Init KafkaInstance
* @type {}
*/
const kafka = core.getKafkaAvro();
kafka.init().then(() => __awaiter(this, void 0, void 0, function* () {
// Wait for the promises to resolve
try {
const extractorProducer = yield kafka.getProducer();
const consumerStream = yield kafka.getConsumerStream();
core.getLogger().info('Producer and consumer are ready to go');
/**
* On data (flow mode) transform data and send them to the data topic
*/
consumerStream
.pipe(new reporter_counter_1.default({ objectMode: true }, core.getReporter(10000), 'read'))
.pipe(new crawler_1.default({ objectMode: true }, core.getLogger()))
.pipe(new reporter_counter_1.default({ objectMode: true }, core.getReporter(), 'write'))
.pipe(new stream_producer_preparer_1.default({ objectMode: true, topic: core.getNconf().get('producer:topics'), partition: (chunk) => chunk.provider }))
.pipe(new debug_1.default({ objectMode: true }, ''))
.pipe(new ProducerStream(extractorProducer, { objectMode: true, topic: core.getNconf().get('producer:topics') }));
core.getLogger().info('Pipes are registered');
}
catch (error) {
core.getLogger().error('Error in retrieving consumer or producer');
core.getLogger().error(error);
}
}))
.catch((err) => {
core.getLogger().error('Its not possible to connect to kafka');
throw err;
});
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7OztBQUFBLHVEQUFzQztBQUN0QyxvRkFBNEU7QUFDNUUsb0dBQTJGO0FBQzNGLHVDQUFnQztBQUNoQyxtRUFBb0U7QUFDcEUsOERBQXVEO0FBRXZEOztFQUVFO0FBQ0YsTUFBTSxJQUFJLEdBQUcsdUJBQUksQ0FBQyxJQUFJLENBQUMsUUFBUSxFQUFFLFNBQVMsR0FBRyxpQkFBaUIsQ0FBQyxDQUFDO0FBRWhFOzs7R0FHRztBQUNILE1BQU0sS0FBSyxHQUFHLElBQUksQ0FBQyxZQUFZLEVBQUUsQ0FBQztBQUNsQyxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsSUFBSSxDQUFDLEdBQVMsRUFBRTtJQUN6QixtQ0FBbUM7SUFDbkMsSUFBSTtRQUNBLE1BQU0saUJBQWlCLEdBQUcsTUFBTSxLQUFLLENBQUMsV0FBVyxFQUFFLENBQUM7UUFDcEQsTUFBTSxjQUFjLEdBQUcsTUFBTSxLQUFLLENBQUMsaUJBQWlCLEVBQUUsQ0FBQztRQUV2RCxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxDQUFDLHVDQUF1QyxDQUFDLENBQUM7UUFFL0Q7O1dBRUc7UUFDSCxjQUFjO2FBQ1QsSUFBSSxDQUFDLElBQUksMEJBQWUsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxJQUFJLENBQUMsV0FBVyxDQUFDLEtBQUssQ0FBRyxFQUFFLE1BQU0sQ0FBQyxDQUFDO2FBQ2hGLElBQUksQ0FBQyxJQUFJLGlCQUFPLENBQUMsRUFBQyxVQUFVLEVBQUUsSUFBSSxFQUFDLEVBQUUsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDLENBQUM7YUFDdkQsSUFBSSxDQUFDLElBQUksMEJBQWUsQ0FBQyxFQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUMsRUFBRSxJQUFJLENBQUMsV0FBVyxFQUFFLEVBQUUsT0FBTyxDQUFDLENBQUM7YUFDMUUsSUFBSSxDQUFDLElBQUksa0NBQXNCLENBQUMsRUFBQyxVQUFVLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxJQUFJLENBQUMsUUFBUSxFQUFFLENBQUMsR0FBRyxDQUFDLGlCQUFpQixDQUFDLEVBQUUsU0FBUyxFQUFFLENBQUMsS0FBSyxFQUFFLEVBQUUsQ0FBQyxLQUFLLENBQUMsUUFBUSxFQUFDLENBQUMsQ0FBQzthQUN6SSxJQUFJLENBQUMsSUFBSSxlQUFLLENBQUMsRUFBQyxVQUFVLEVBQUUsSUFBSSxFQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7YUFDdkMsSUFBSSxDQUFDLElBQUksY0FBYyxDQUFDLGlCQUFpQixFQUFFLEVBQUMsVUFBVSxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLEdBQUcsQ0FBQyxpQkFBaUIsQ0FBQyxFQUFDLENBQUMsQ0FBQyxDQUFDO1FBRXBILElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLENBQUMsc0JBQXNCLENBQUMsQ0FBQztLQUNqRDtJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ1osSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDLEtBQUssQ0FBQywwQ0FBMEMsQ0FBQyxDQUFDO1FBQ25FLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLENBQUM7S0FDakM7QUFFTCxDQUFDLENBQUEsQ0FBQztLQUNHLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO0lBQ1gsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDLEtBQUssQ0FBQyxzQ0FBc0MsQ0FBQyxDQUFDO0lBQy9ELE1BQU0sR0FBRyxDQUFDO0FBQ2QsQ0FBQyxDQUFDLENBQUMifQ==
\ No newline at end of file
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
const request = require("request");
const cheerio_1 = require("cheerio");
const url_1 = require("url");
/**
* The returning document
* @param url
* @param res
* @constructor
*/
function Document(url, res) {
this.url = url;
this.res = res;
}
/**
* Properties of the document
*/
Document.prototype = {
constructor: Document,
// Lazy parse
get $() {
return this._$ || (this._$ = cheerio_1.default.load(this.res.body));
},
resolve: function (uri) {
return url_1.resolve(this.url, uri);
}
};
/**
* MagpieTransformer
*/
class MagpieTransformer extends stream_1.Transform {
constructor(options) {
super(options);
/**
* Timeout for a server
*/
this.serverTimeout = 30 * 1000;
/**
* delay between two requests
*/
this.delay = 200;
/**
* Options for request
*/
this.requestOptions = {};
/**
* A counter for pending requests
*/
this.pending = 0;
/**
* the concurrency of the crawler
*/
this.concurrent = 10;
this.logger = options.logger;
this.delay = options.delay || 200;
this.concurrent = options.concurrent || 10;
this.requestOptions = options.requestOptions || {};
}
/**
* @inheritDoc
* @param chunk
* @param encoding
* @param callback
* @private
*/
_transform(chunk, encoding, callback) {
let data;
if (Buffer.isBuffer(chunk)) {
const buffer = Buffer.from(chunk).toString('utf8');
data = JSON.parse(buffer);
}
else {
data = JSON.parse(chunk);
}
if (this.pending >= this.concurrent) {
return this.once('free', () => {
this._transform(chunk, encoding, callback);
});
}
this.pending++;
setTimeout(() => {
let foundServer = false;
// if the server is not reacting wait for a timeout and then call the callback
setTimeout(() => {
this.pending--;
if (!foundServer) {
this.logger.warn('Too fast. ServerTimeout triggered');
this.emit('free');
}
}, this.serverTimeout);
const options = Object.assign(Object.assign({}, this.requestOptions), { url: data.url });
MagpieTransformer._request(options, (err, res, _) => {
this.pending--;
foundServer = true;
if (err) {
this.logger.error(err, data.url);
this.push(null);
}
const doc = new Document(data.url, res);
this.logger.info('Success', data.url);
this.push(doc);
this.emit('free');
});
}, this.delay);
callback();
}
// Wrap it for easier mocking
static _request(opts, done) {
// All options forwarded to request()
request(opts, done);
}
end(chunk, enc, callback) {
if (this.pending) {
return this.once('free', () => {
this.end(chunk, enc, callback);
});
}
if (typeof chunk === 'function') {
callback = chunk;
chunk = null;
}
if (typeof enc === 'function') {
callback = enc;
enc = null;
}
if (chunk) {
this.write(chunk, enc);
return this.once('free', () => {
this.end(callback);
});
}
if (callback)
this.on('finish', callback);
super.end();
}
}
exports.default = MagpieTransformer;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWFncGllLXRyYW5zZm9ybWVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL21hZ3BpZS10cmFuc2Zvcm1lci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOztBQUFBLG1DQUFpQztBQUVqQyxtQ0FBbUM7QUFDbkMscUNBQThCO0FBQzlCLDZCQUE0QjtBQUU1Qjs7Ozs7R0FLRztBQUNILFNBQVMsUUFBUSxDQUFDLEdBQUcsRUFBRSxHQUFHO0lBQ3RCLElBQUksQ0FBQyxHQUFHLEdBQUcsR0FBRyxDQUFDO0lBQ2YsSUFBSSxDQUFDLEdBQUcsR0FBRyxHQUFHLENBQUM7QUFDbkIsQ0FBQztBQUVEOztHQUVHO0FBQ0gsUUFBUSxDQUFDLFNBQVMsR0FBRztJQUNqQixXQUFXLEVBQUUsUUFBUTtJQUVyQixhQUFhO0lBQ2IsSUFBSSxDQUFDO1FBQ0QsT0FBTyxJQUFJLENBQUMsRUFBRSxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsR0FBRyxpQkFBTyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7SUFDOUQsQ0FBQztJQUVELE9BQU8sRUFBRSxVQUFTLEdBQUc7UUFDakIsT0FBTyxhQUFPLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxHQUFHLENBQUMsQ0FBQztJQUNsQyxDQUFDO0NBQ0osQ0FBQztBQUVGOztHQUVHO0FBQ0gsTUFBTSxpQkFBa0IsU0FBUSxrQkFBUztJQStCckMsWUFBWSxPQUFPO1FBQ2YsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBOUJuQjs7V0FFRztRQUNLLGtCQUFhLEdBQUcsRUFBRSxHQUFHLElBQUksQ0FBQztRQU1sQzs7V0FFRztRQUNjLFVBQUssR0FBVyxHQUFHLENBQUM7UUFFckM7O1dBRUc7UUFDYyxtQkFBYyxHQUFHLEVBQUUsQ0FBQztRQUVyQzs7V0FFRztRQUNLLFlBQU8sR0FBVyxDQUFDLENBQUM7UUFFNUI7O1dBRUc7UUFDYyxlQUFVLEdBQVcsRUFBRSxDQUFDO1FBSXJDLElBQUksQ0FBQyxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQztRQUM3QixJQUFJLENBQUMsS0FBSyxHQUFHLE9BQU8sQ0FBQyxLQUFLLElBQUksR0FBRyxDQUFDO1FBQ2xDLElBQUksQ0FBQyxVQUFVLEdBQUcsT0FBTyxDQUFDLFVBQVUsSUFBSSxFQUFFLENBQUM7UUFDM0MsSUFBSSxDQUFDLGNBQWMsR0FBRyxPQUFPLENBQUMsY0FBYyxJQUFJLEVBQUUsQ0FBQztJQUN2RCxDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ0ksVUFBVSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUTtRQUN2QyxJQUFJLElBQUksQ0FBQztRQUNULElBQUksTUFBTSxDQUFDLFFBQVEsQ0FBQyxLQUFLLENBQUMsRUFBRTtZQUN4QixNQUFNLE1BQU0sR0FBRyxNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUNuRCxJQUFJLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQztTQUM3QjthQUFNO1lBQ0gsSUFBSSxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLENBQUM7U0FDNUI7UUFFRCxJQUFJLElBQUksQ0FBQyxPQUFPLElBQUksSUFBSSxDQUFDLFVBQVUsRUFBRTtZQUNqQyxPQUFPLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxFQUFFLEdBQUcsRUFBRTtnQkFDMUIsSUFBSSxDQUFDLFVBQVUsQ0FBQyxLQUFLLEVBQUUsUUFBUSxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQy9DLENBQUMsQ0FBQyxDQUFDO1NBQ047UUFFRCxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDZixVQUFVLENBQUMsR0FBRyxFQUFFO1lBQ1osSUFBSSxXQUFXLEdBQUcsS0FBSyxDQUFDO1lBQ3hCLDhFQUE4RTtZQUM5RSxVQUFVLENBQUMsR0FBRyxFQUFFO2dCQUNaLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQztnQkFDZixJQUFJLENBQUMsV0FBVyxFQUFFO29CQUNkLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLG1DQUFtQyxDQUFDLENBQUM7b0JBQ3RELElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLENBQUM7aUJBQ3JCO1lBQ0wsQ0FBQyxFQUFFLElBQUksQ0FBQyxhQUFhLENBQUMsQ0FBQztZQUV2QixNQUFNLE9BQU8sbUNBQU8sSUFBSSxDQUFDLGNBQWMsR0FBSyxFQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsR0FBRyxFQUFDLENBQUMsQ0FBQztZQUM3RCxpQkFBaUIsQ0FBQyxRQUFRLENBQUMsT0FBTyxFQUFHLENBQUMsR0FBRyxFQUFFLEdBQUcsRUFBRSxDQUFDLEVBQUUsRUFBRTtnQkFDakQsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO2dCQUVmLFdBQVcsR0FBRyxJQUFJLENBQUM7Z0JBQ25CLElBQUksR0FBRyxFQUFFO29CQUNMLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7b0JBQ2pDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7aUJBQ25CO2dCQUVELE1BQU0sR0FBRyxHQUFHLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxDQUFDLENBQUM7Z0JBQ3hDLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQ3RDLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQ2YsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQTtZQUNyQixDQUFDLENBQUMsQ0FBQztRQUNQLENBQUMsRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDZixRQUFRLEVBQUUsQ0FBQztJQUNmLENBQUM7SUFFRCw2QkFBNkI7SUFDckIsTUFBTSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsSUFBSTtRQUM5QixxQ0FBcUM7UUFDckMsT0FBTyxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQztJQUN4QixDQUFDO0lBT00sR0FBRyxDQUFDLEtBQVUsRUFBRSxHQUFXLEVBQUUsUUFBYTtRQUM3QyxJQUFJLElBQUksQ0FBQyxPQUFPLEVBQUU7WUFDZCxPQUFPLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxFQUFFLEdBQUcsRUFBRTtnQkFDMUIsSUFBSSxDQUFDLEdBQUcsQ0FBQyxLQUFLLEVBQUUsR0FBRyxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQ25DLENBQUMsQ0FBQyxDQUFDO1NBQ047UUFFRCxJQUFJLE9BQU8sS0FBSyxLQUFLLFVBQVUsRUFBRTtZQUM3QixRQUFRLEdBQUcsS0FBSyxDQUFDO1lBQ2pCLEtBQUssR0FBRyxJQUFJLENBQUM7U0FDaEI7UUFFRCxJQUFJLE9BQU8sR0FBRyxLQUFLLFVBQVUsRUFBRTtZQUMzQixRQUFRLEdBQUcsR0FBRyxDQUFDO1lBQ2YsR0FBRyxHQUFHLElBQUksQ0FBQztTQUNkO1FBRUQsSUFBSSxLQUFLLEVBQUU7WUFDUCxJQUFJLENBQUMsS0FBSyxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUMsQ0FBQztZQUN2QixPQUFPLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxFQUFFLEdBQUcsRUFBRTtnQkFDMUIsSUFBSSxDQUFDLEdBQUcsQ0FBQyxRQUFRLENBQUMsQ0FBQztZQUN2QixDQUFDLENBQUMsQ0FBQztTQUNOO1FBRUQsSUFBSSxRQUFRO1lBQUUsSUFBSSxDQUFDLEVBQUUsQ0FBQyxRQUFRLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFFMUMsS0FBSyxDQUFDLEdBQUcsRUFBRSxDQUFDO0lBQ2hCLENBQUM7Q0FDSjtBQUVELGtCQUFlLGlCQUFpQixDQUFDIn0=
\ No newline at end of file
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
class KafkaProducerWritable extends stream_1.Writable {
constructor(options = {}, producer, nconf, reporter) {
super(options);
this.timeout = 500;
this.producer = producer;
this.nconf = nconf;
this.reporter = reporter;
}
_write(chunk, encoding, callback) {
return __awaiter(this, void 0, void 0, function* () {
try {
// produce to a dynamic topic
const data = {
topic: 'extractor',
value: chunk,
partition: chunk.provider,
};
yield this.producer.produce(this.nconf.get('topic:config:topic'), this.nconf.get('topic:config:partion'), data);
this.reporter.setDataOut(1);
callback();
}
catch (err) {
// handle backpressure if the queue is full
// if (CODES.ERRORS.ERR__QUEUE_FULL === err.code) {
// setTimeout(() => {
// console.
// this._write(chunk, encoding, callback);
//
// }, this.timeout);
// } else {
setTimeout(() => {
callback(err);
}, this.timeout);
// }
}
});
}
}
exports.default = KafkaProducerWritable;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvZHVjZXItd3JpdGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL3Byb2R1Y2VyLXdyaXRlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7O0FBQUEsbUNBQWdDO0FBSWhDLDJCQUE0QixTQUFRLGlCQUFRO0lBS3hDLFlBQVksT0FBTyxHQUFHLEVBQUUsRUFBRSxRQUFrQixFQUFFLEtBQUssRUFBRSxRQUEyQjtRQUNoRixLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7UUFIUCxZQUFPLEdBQUcsR0FBRyxDQUFDO1FBSWxCLElBQUksQ0FBQyxRQUFRLEdBQUcsUUFBUSxDQUFDO1FBQ3pCLElBQUksQ0FBQyxLQUFLLEdBQUcsS0FBSyxDQUFDO1FBQ25CLElBQUksQ0FBQyxRQUFRLEdBQUcsUUFBUSxDQUFDO0lBQzdCLENBQUM7SUFFWSxNQUFNLENBQUMsS0FBSyxFQUFFLFFBQVEsRUFBRSxRQUFROztZQUN6QyxJQUFJO2dCQUNBLDZCQUE2QjtnQkFDN0IsTUFBTSxJQUFJLEdBQUc7b0JBQ1QsS0FBSyxFQUFFLFdBQVc7b0JBQ2xCLEtBQUssRUFBRSxLQUFLO29CQUNaLFNBQVMsRUFBRSxLQUFLLENBQUMsUUFBUTtpQkFDNUIsQ0FBQTtnQkFDRCxNQUFNLElBQUksQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLG9CQUFvQixDQUFDLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsc0JBQXNCLENBQUMsRUFBRSxJQUFJLENBQUMsQ0FBQztnQkFDaEgsSUFBSSxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDLENBQUM7Z0JBQzVCLFFBQVEsRUFBRSxDQUFDO2FBQ2Q7WUFBQyxPQUFPLEdBQUcsRUFBRTtnQkFDZCwyQ0FBMkM7Z0JBQzNDLG1EQUFtRDtnQkFDbkQsdUJBQXVCO2dCQUNILGVBQWU7Z0JBQ2YsOENBQThDO2dCQUM5QyxFQUFFO2dCQUNGLHNCQUFzQjtnQkFDMUMsWUFBWTtnQkFDWixVQUFVLENBQUMsR0FBRyxFQUFFO29CQUNELFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQztnQkFDZCxDQUFDLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDO2dCQUNoQyxJQUFJO2FBQ0g7UUFDTCxDQUFDO0tBQUE7Q0FDSjtBQUVELGtCQUFlLHFCQUFxQixDQUFDIn0=
\ No newline at end of file
......@@ -108,9 +108,9 @@
"dev": true
},
"@types/node": {
"version": "9.6.52",
"resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.52.tgz",
"integrity": "sha512-d6UdHtc8HKe3NTruj9mHk2B8EiHZyuG/00aYbUedHvy9sBhtLAX1gaxSNgvcheOvIZavvmpJYlwfHjjxlU/Few=="
"version": "10.14.21",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.21.tgz",
"integrity": "sha512-nuFlRdBiqbF+PJIEVxm2jLFcQWN7q7iWEJGsBV4n7v1dbI9qXB8im2pMMKMCUZe092sQb5SQft2DHfuQGK5hqQ=="
},
"@types/request": {
"version": "2.48.1",
......@@ -786,6 +786,18 @@
"through2": "^2.0.3",
"typescript": "^2.8.3",
"winston": "^2.4.2"
},
"dependencies": {
"@types/node": {
"version": "9.6.52",
"resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.52.tgz",
"integrity": "sha512-d6UdHtc8HKe3NTruj9mHk2B8EiHZyuG/00aYbUedHvy9sBhtLAX1gaxSNgvcheOvIZavvmpJYlwfHjjxlU/Few=="
},
"typescript": {
"version": "2.9.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-2.9.2.tgz",
"integrity": "sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w=="
}
}
},
"get-caller-file": {
......@@ -2161,9 +2173,9 @@
"dev": true
},
"typescript": {
"version": "2.9.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-2.9.2.tgz",
"integrity": "sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w=="
"version": "3.6.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.4.tgz",
"integrity": "sha512-unoCll1+l+YK4i4F8f22TaNVPRHcD9PA3yCuZ8g5e0qGqlVlJ/8FSateOLLSagn+Yg5+ZwuPkL8LFUc0Jcvksg=="
},
"uri-js": {
"version": "4.2.2",
......
......@@ -15,17 +15,17 @@
"author": "",
"license": "ISC",
"dependencies": {
"@types/node": "^9.6.52",
"@types/node": "^10.14",
"concurrently": "^3.5.1",
"geolinker-common": "git+https://gitlab+deploy-token-1:vnsdCm_t84QGVA2U4kw4@source.dodis.ch/histhub/geolinker-common.git",
"source-map": "^0.7.3",
"typescript": "^2.8.3",
"typescript": "^3.6",
"cheerio": "^0.22"
},
"devDependencies": {
"@types/chai": "^4.2.3",
"@types/mocha": "^5.2.7",
"chai": "^4.2.0",
"@types/chai": "^4.2",
"@types/mocha": "^5.2",
"chai": "^4.2",
"mocha": "^6",
"sinon": "^6.3.5",
"ts-node": "^8.4",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment