Commit 67e3867c authored by tobinski's avatar tobinski

Squashed commit of the following:

commit a506da316d40ce3176acc35234d421fde2bcca35
Author: tobinski <tobinski@dodis.ch>
Date:   Tue Jan 7 09:19:44 2020 +0100

    Update test

commit 086f52820f6ddf0470e3e06b5a32614633ccdb81
Author: tobinski <tobinski@dodis.ch>
Date:   Tue Jan 7 09:04:47 2020 +0100

    Fix bug with too many calls of all pages

commit 15e3bf5c
Author: tobinski <tobinski@dodis.ch>
Date:   Tue Nov 19 12:27:40 2019 +0100

    Remove debug
parent 0f170631
......@@ -12,12 +12,19 @@ class SitemapReader extends Readable {
private logger: LoggerInstance;
/**
* Simple flag to decide if we just started or nor
* Headers for magpie
*/
private initial = true;
private static headers = { 'user-agent': 'magpie' , 'content-type': 'application/json'};
/**
* Initial page
*/
private counter = 0;
/**
* Initial total
*/
private total = 0;
/**
* Start url for dodis
*/
......@@ -34,9 +41,27 @@ class SitemapReader extends Readable {
* @private
*/
public _read(size) {
SitemapReader._request({url: this.startUrl}, (err, res, _) => {
this.handleRequest(err, res);
});
// check if we are at the end
if(this.counter <= this.total) {
SitemapReader._request({url: `${this.startUrl}&p=${this.counter}`}, (err, res, _) => {
this.counter++;
if (err) {
this.logger.error(err);
return this.push(err);
}
if (res) {
const data = JSON.parse(res.body);
this.total = data.totalPages;
for (let doc of data.data) {
this.push(JSON.stringify({url: `https://dodis.ch/G${doc.id}`}));
}
} else {
this.push(null);
}
});
} else {
this.push(null);
}
}
/**
......@@ -50,33 +75,6 @@ class SitemapReader extends Readable {
opts = {...{headers: SitemapReader.headers}, ...opts};
request(opts, done);
}
/**
* Handle the request and push data
* @param res
*/
private handleRequest(err, res) {
if(err) {
this.logger.error(err);
return this.push(null);
}
if(res) {
const data = JSON.parse(res.body);
if (this.initial) {
for (let i = 1; i < data.totalPages; i++) {
setTimeout(() => {
SitemapReader._request({url: `${this.startUrl}&p=${i}`}, this.handleRequest.bind(this));
}, 300 * i);
}
this.initial = false;
}
for (let doc of data.data) {
this.push(JSON.stringify({url: `https://dodis.ch/G${doc.id}`}));
}
} else {
this.push(null);
}
}
}
export default SitemapReader;
......@@ -38,6 +38,7 @@ describe('Test dodis-normalizer-transformer', () => {
alternative_name: null,
country: null,
location: `47.497913,19.040236`,
typology: []
};
// we cant predict modification_date
delete data.modification_date;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment