Initial commit
parents
Showing
.gitignore
0 → 100644
.gitlab-ci.yml
0 → 100644
Dockerfile
0 → 100644
Readme.md
0 → 100644
config.json
0 → 100644
package-lock.json
0 → 100644
This diff is collapsed.
package.json
0 → 100644
{ | ||
"name": "magpie", | ||
"version": "1.0.0", | ||
"description": "Magpie get urls from kafka and fetchs them in the wild of the internet", | ||
"main": "dist/index.js", | ||
"types": "dist/index.d.js", | ||
"scripts": { | ||
"test": "node_modules/.bin/mocha -r ts-node/register test/**/*.spec.ts", | ||
"lint": "node_modules/.bin/tslint -c tslint.json 'src/**/*.ts'", | ||
"build": "node_modules/.bin/tsc", | ||
"watch": "concurrently -k -p \"[{name}]\" -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold,green.bold\" \"npm run watch-ts\" \"npm run watch-node\"", | ||
"watch-node": "nodemon dist/index.js", | ||
"watch-ts": "tsc -w" | ||
}, | ||
"author": "", | ||
"license": "ISC", | ||
"dependencies": { | ||
"@types/node": "^9.6.18", | ||
"bunyan-format": "^0.2.1", | ||
"concurrently": "^3.5.1", | ||
"geolinker-common": "git+https://gitlab+deploy-token-1:vnsdCm_t84QGVA2U4kw4@source.dodis.ch/histhub/geolinker-common.git", | ||
"node-cleanup": "^2.1.2", | ||
"node-spider": "^1.4.1", | ||
"source-map": "^0.7.3", | ||
"typescript": "^2.8.3" | ||
}, | ||
"devDependencies": { | ||
"ts-node": "^6.1.0", | ||
"tslint": "^5.9.1" | ||
} | ||
} |
src/Extractor.ts
0 → 100644
src/crawler.ts
0 → 100644
src/index.ts
0 → 100644
src/sitemap-generator.ts
0 → 100644
test/.gitkeep
0 → 100644
tsconfig.json
0 → 100644