Skip to content
Snippets Groups Projects

Resolve "Add ortsnamen collector"

Merged Tobinsk requested to merge 8-add-ortsnamen-collector-2 into main
26 files
+ 4475
30
Compare changes
  • Side-by-side
  • Inline
Files
26
+ 35
0
import Crawler from 'api-crawler-lib';
import {createElasticsearchWriter} from 'es-sink/dist';
import {Core} from 'geolinker-common';
import {createNeo4jStreamWriter} from 'linker/dist';
import OrtsnamenAnalyserTransformer from './ortsnamen-analyser-transformer';
import OrtsnamenNormalizerTransformer from './ortsnamen-normalizer-transformer';
/**
* load sourcemap and config
*/
const core = Core.init('ortsnamen', __dirname + '/../config.json');
const crawlStreamConsumer = new Crawler({
objectMode: true,
paramLimitName: 'limit',
paramOffsetName: 'offset',
limit: 500,
dataPath: 'results',
logger: core.getLogger(),
baseUrl: 'https://search.ortsnamen.ch/api/toponyms?q=*',
});
/**
* On data (flow mode) normalize data from crawler and send them to the geolinker
*/
crawlStreamConsumer
.pipe(new OrtsnamenNormalizerTransformer({objectMode: true, logger: core.getLogger()}))
.pipe(createElasticsearchWriter(core));
/**
* On data (flow mode) extract data from crawler and send them to liker
*/
crawlStreamConsumer
.pipe(new OrtsnamenAnalyserTransformer({objectMode: true, author: core.getNconf().get('producer:linker:author')}))
.pipe(createNeo4jStreamWriter(core));
Loading