diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a752ec56892463c31e78fe48fdf2b7050423135a..011e4362f387d6b1db052aaa13965c447843adc5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,34 +1,23 @@ +image: docker:latest + services: - docker:dind -variables: - IMAGE_TAG: $CI_REGISTRY/$CI_IMAGE:$CI_COMMIT_REF_NAME - LATEST_TAG: $CI_REGISTRY/$CI_IMAGE:latest - -cache: - paths: - - node_modules/ +before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY -build: +build-master: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE" . + - docker push "$CI_REGISTRY_IMAGE" only: - master - image: docker:latest - services: - - docker:dind + +build: stage: build script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker build -t $IMAGE_TAG . - - docker tag $IMAGE_TAG $LATEST_TAG - - docker push $IMAGE_TAG - - docker push $LATEST_TAG - -test: - stage: test + - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" only: - - sowhereinthefuture - image: $LATEST_TAG - before_script: - - export NODE_ENV=dev && npm install - script: - - npm run test \ No newline at end of file + - tags diff --git a/Dockerfile b/Dockerfile index afb08a36de70f4c39eea1082c1f9a4eb40d470f6..2bad67b45aebbf6ef1ce2eef85257cf5f5168695 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,16 @@ -FROM node:8-alpine +FROM node:10-slim # add basic libs -RUN apk --no-cache add \ +RUN apt-get update && apt-get install -y \ bash \ g++ \ + gcc \ ca-certificates \ - lz4-dev \ - musl-dev \ - cyrus-sasl-dev \ - openssl-dev \ make \ python \ bash \ git -RUN apk add --no-cache --virtual .build-deps gcc zlib-dev libc-dev bsd-compat-headers py-setuptools libexecinfo libexecinfo-dev - # Create app directory RUN mkdir -p /usr/local/app @@ -34,4 +29,4 @@ RUN npm install RUN npm run build # run app -CMD node dist/index.js \ No newline at end of file +CMD node dist/index.js diff --git a/package-lock.json b/package-lock.json index 44e5f4e40b7fc407bf897cad6e7dd078d9ab47a8..7f5da98264739410c11eaec1051b1f8a31fd7ff6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "@types/caseless": { - "version": "0.12.1", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz", - "integrity": "sha512-FhlMa34NHp9K5MY1Uz8yb+ZvuX0pnvn3jScRSNAb75KHGB8d3rEU6hqMs3Z2vjuytcMfRg6c5CHMc3wtYyD2/A==" + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", + "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" }, "@types/es6-promise": { "version": "3.3.0", @@ -31,9 +31,9 @@ "integrity": "sha512-LMSOxMKNJ8tGqUVs8lSIT8RGo1XGWYada/ZU2QZcbcD6AW9futXDE99tfQA0K6DK60GXcwplsGGK5KABRmI5GA==" }, "@types/request": { - "version": "2.47.1", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.47.1.tgz", - "integrity": "sha512-TV3XLvDjQbIeVxJ1Z3oCTDk/KuYwwcNKVwz2YaT0F5u86Prgc4syDAp6P96rkTQQ4bIdh+VswQIC9zS6NjY7/g==", + "version": "2.48.1", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz", + "integrity": "sha512-ZgEZ1TiD+KGA9LiAAPPJL68Id2UWfeSO62ijSXZjFJArVV+2pKcsVHmrcu+1oiE3q6eDGiFiSolRc4JHoerBBg==", "requires": { "@types/caseless": "*", "@types/form-data": "*", @@ -42,27 +42,19 @@ } }, "@types/tough-cookie": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.3.tgz", - "integrity": "sha512-MDQLxNFRLasqS4UlkWMSACMKeSm1x4Q3TxzUC7KQUsh6RK1ZrQ0VEyE3yzXcBu+K8ejVj4wuX32eUG02yNp+YQ==" - }, - "@types/winston": { - "version": "2.3.9", - "resolved": "https://registry.npmjs.org/@types/winston/-/winston-2.3.9.tgz", - "integrity": "sha512-zzruYOEtNgfS3SBjcij1F6HlH6My5n8WrBNhP3fzaRM22ba70QBC2ATs18jGr88Fy43c0z8vFJv5wJankfxv2A==", - "requires": { - "@types/node": "*" - } + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", + "integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg==" }, "ajv": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", - "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", "requires": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", + "fast-deep-equal": "^2.0.1", "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" } }, "ansi-regex": { @@ -126,9 +118,9 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "avsc": { - "version": "5.4.2", - "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.2.tgz", - "integrity": "sha512-nXfk2U3E/CtPRm05I81q8VUzMvSD7JYejjbE1KiMdqdsmcFx+fs1fv0IUj5yFIloBc+fJy5KGE3hfcZEiOKZLQ==" + "version": "5.4.10", + "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.10.tgz", + "integrity": "sha512-6/s4d/o0EshemF2UlVbdMabOQ3Mavcd2okWmhfQlfOFVMq1kFaGtIlIYK4WPtc69EfkdcjElIti951bdRLdBNA==" }, "aws-sign2": { "version": "0.7.0", @@ -141,12 +133,12 @@ "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "axios": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", - "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", + "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", "requires": { - "follow-redirects": "^1.3.0", - "is-buffer": "^1.1.5" + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" } }, "babel-code-frame": { @@ -196,15 +188,17 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "optional": true, "requires": { "tweetnacl": "^0.14.3" } }, "bindings": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz", - "integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "requires": { + "file-uri-to-path": "1.0.0" + } }, "bl": { "version": "1.0.3", @@ -240,9 +234,9 @@ } }, "bluebird": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", - "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" + "version": "3.5.5", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz", + "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==" }, "boolbase": { "version": "1.0.0", @@ -341,7 +335,7 @@ "dependencies": { "lodash": { "version": "3.10.1", - "resolved": "http://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" } } @@ -361,11 +355,6 @@ "wrap-ansi": "^2.0.0" } }, - "co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" - }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -554,7 +543,6 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "optional": true, "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -574,9 +562,9 @@ } }, "es6-promise": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz", - "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ==" + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" }, "escape-string-regexp": { "version": "1.0.5", @@ -610,21 +598,26 @@ "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" }, "fast-deep-equal": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", - "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" }, "fast-json-stable-stringify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "follow-redirects": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.5.tgz", - "integrity": "sha512-GHjtHDlY/ehslqv0Gr5N0PUJppgg/q0rOBvX0na1s7y1A3LWxPqCYU76s3Z1bM4+UZB4QF0usaXLT5wFpof5PA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", "requires": { - "debug": "^3.1.0" + "debug": "=3.1.0" } }, "forever-agent": { @@ -633,12 +626,12 @@ "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", - "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { "asynckit": "^0.4.0", - "combined-stream": "1.0.6", + "combined-stream": "^1.0.6", "mime-types": "^2.1.12" } }, @@ -665,19 +658,18 @@ } }, "geolinker-common": { - "version": "git+https://gitlab+deploy-token-1:vnsdCm_t84QGVA2U4kw4@source.dodis.ch/histhub/geolinker-common.git#ba026ae5b941009a2b27a5f2cd6de5ddafba8a90", + "version": "git+https://gitlab+deploy-token-1:vnsdCm_t84QGVA2U4kw4@source.dodis.ch/histhub/geolinker-common.git#aa62da3c00ed5bc359c986ee95bb418bf708afb8", "from": "git+https://gitlab+deploy-token-1:vnsdCm_t84QGVA2U4kw4@source.dodis.ch/histhub/geolinker-common.git", "requires": { "@types/es6-promise": "^3.3.0", "@types/node": "^9.6.18", "@types/request": "^2.47.0", - "@types/winston": "^2.3.9", "axios": "^0.18.0", "bunyan-format": "^0.2.1", "concurrently": "^3.5.1", "cpu-stat": "^2.0.1", "js-yaml": "^3.11.0", - "kafka-avro": "^1.0.2", + "kafka-avro": "^1.2.0", "nconf": "^0.10.0", "ramda": "^0.25.0", "request": "^2.87.0", @@ -719,11 +711,11 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.0.tgz", - "integrity": "sha512-+qnmNjI4OfH2ipQ9VQOw23bBd/ibtfbVdK2fYbY4acTDqKTW/YDp9McimZdDbG8iV9fZizUqQMD5xvriB146TA==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { - "ajv": "^5.3.0", + "ajv": "^6.5.5", "har-schema": "^2.0.0" } }, @@ -850,9 +842,9 @@ "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" }, "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==" }, "is-builtin-module": { "version": "1.0.0", @@ -932,8 +924,7 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-parse-better-errors": { "version": "1.0.2", @@ -946,9 +937,9 @@ "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stringify-safe": { "version": "5.0.1", @@ -972,9 +963,9 @@ } }, "kafka-avro": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/kafka-avro/-/kafka-avro-1.1.0.tgz", - "integrity": "sha512-NnNGruEeWLyczcY80rC1RemxD0A0JXz4r2BdzZCRuRb8xcWQu9cideDnoXdBOFG9slB/b022kNUgsGFcBWUL5A==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/kafka-avro/-/kafka-avro-1.2.0.tgz", + "integrity": "sha512-Vhza9re6zGJ8NbA0nlM/sH3LQSPUhSqNQ6QlFQWnZzZbaenl5GehmEHmCQ1U/hnK4wWfaTsw2iIlx98zmd1gPw==", "requires": { "avsc": "^5.2.3", "axios": "^0.15.3", @@ -1076,9 +1067,9 @@ } }, "moment": { - "version": "2.22.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz", - "integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y=", + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==", "optional": true }, "ms": { @@ -1098,9 +1089,9 @@ } }, "nan": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", - "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", + "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==" }, "nconf": { "version": "0.10.0", @@ -1171,7 +1162,7 @@ }, "bluebird": { "version": "2.11.0", - "resolved": "http://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=" }, "caseless": { @@ -1181,7 +1172,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -1244,7 +1235,7 @@ }, "request": { "version": "2.61.0", - "resolved": "http://registry.npmjs.org/request/-/request-2.61.0.tgz", + "resolved": "https://registry.npmjs.org/request/-/request-2.61.0.tgz", "integrity": "sha1-aXPLKslIhfAmk/VU7sZEgdYBP58=", "requires": { "aws-sign2": "~0.5.0", @@ -1561,9 +1552,9 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "sshpk": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.2.tgz", - "integrity": "sha1-xvxhZIo9nE52T9P8306hBeSSupg=", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -1633,11 +1624,11 @@ } }, "through2": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.3.tgz", - "integrity": "sha1-AARWmzfHx0ujnEPzzteNGtlBQL4=", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", "requires": { - "readable-stream": "^2.1.5", + "readable-stream": "~2.3.6", "xtend": "~4.0.1" }, "dependencies": { @@ -1754,14 +1745,28 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "typescript": { "version": "2.9.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.9.2.tgz", "integrity": "sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==" }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + }, + "dependencies": { + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -1797,9 +1802,9 @@ "integrity": "sha1-+OGqHuWlPsW/FR/6CXQqatdpeHY=" }, "winston": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.3.tgz", - "integrity": "sha512-GYKuysPz2pxYAVJD2NPsDLP5Z79SDEzPm9/j4tCjkF/n89iBNGBMJcR+dMUqxgPNgoSs6fVygPi+Vl2oxIpBuw==", + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz", + "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==", "requires": { "async": "~1.0.0", "colors": "1.0.x", diff --git a/src/crawler.ts b/src/crawler.ts index c4dc06bcf429035f3c8e379a3962ff10ab04092d..548166933aad5b908d827313183267073669314d 100644 --- a/src/crawler.ts +++ b/src/crawler.ts @@ -34,7 +34,7 @@ const spider = new Spider({ // How many requests can be run in parallel concurrent: 10, // How long to wait after each request - delay: 100, + delay: 200, // A stream to where internal logs are sent, optional logs: logger, // Re-visit visited URLs, false by default @@ -69,7 +69,10 @@ class Crawler extends Transform { */ private timeout = 20; - private serverTimeout = 15 * 1000; + /** + * Timeout for a server + */ + private serverTimeout = 30 * 1000; /** * Logger */