Commit 64e00417 authored by Tobias Steiner's avatar Tobias Steiner
Browse files

Fix vulk insert

parent 8b04e08c
......@@ -9,7 +9,7 @@ export class ElasticsearchTransformer extends Transform {
* Batch size
* @type {string}
*/
private batchSize: number = 500;
private batchSize: number = 2000;
/**
* Cache for batch
......@@ -34,19 +34,20 @@ export class ElasticsearchTransformer extends Transform {
/**
* Timeout if something went wrong
*/
private timeout: number = 500;
private timeout: number = 1000;
constructor(options: any = {}, core: InterfaceCore) {
super(options);
this.core = core;
this.client = createClient(core.getNconf());
this.timeout = options.timeout || 500;
this.batchSize = options.batchSize || 500;
this.timeout = options.timeout || 1000;
this.batchSize = options.batchSize || 2000;
}
public async _transform(chunk, encoding, callback) {
this.batch.push(this.prepareBulk(chunk.parsed));
this.batch.push(this.prepareMeta(chunk.parsed));
this.batch.push(this.prepareData(chunk.parsed));
if (this.batch.length > this.batchSize) {
try {
await this.client.bulk({body: this.batch});
......@@ -59,13 +60,18 @@ export class ElasticsearchTransformer extends Transform {
setImmediate(callback);
}
private prepareBulk(row) {
const meta: any = {
index: {
_index: this.index,
_id: row.id,
_type: 'geolinker'} };
const data: any = {
private prepareMeta(row) {
return {
index: {
_index: this.index,
_id: row.id,
_type: 'geolinker',
},
};
}
private prepareData(row) {
return {
id: row.id,
provider: row.provider,
url: row.url,
......@@ -78,6 +84,5 @@ export class ElasticsearchTransformer extends Transform {
location: row.location,
modification_date: row.modification_date,
};
return `${JSON.stringify(meta)}${JSON.stringify(data)}`;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment