Commit 530dd578 authored by Tobias Steiner's avatar Tobias Steiner
Browse files

Initial commit

parents
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# next.js build output
.next
services:
- docker:dind
variables:
IMAGE_TAG: $CI_REGISTRY/$CI_IMAGE:$CI_COMMIT_REF_NAME
LATEST_TAG: $CI_REGISTRY/$CI_IMAGE:latest
cache:
paths:
- node_modules/
build:
only:
- master
- tags
image: docker:latest
services:
- docker:dind
stage: build
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGE_TAG .
- docker tag $IMAGE_TAG $LATEST_TAG
- docker push $IMAGE_TAG
- docker push $LATEST_TAG
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<TypeScriptCodeStyleSettings version="0">
<option name="FORCE_SEMICOLON_STYLE" value="true" />
<option name="SPACE_BEFORE_FUNCTION_LEFT_PARENTH" value="false" />
<option name="USE_DOUBLE_QUOTES" value="false" />
<option name="FORCE_QUOTE_STYlE" value="true" />
<option name="SPACE_WITHIN_TYPE_ASSERTION" value="true" />
</TypeScriptCodeStyleSettings>
<codeStyleSettings language="JavaScript">
<indentOptions>
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="2" />
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="TypeScript">
<option name="RIGHT_MARGIN" value="120" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="IF_BRACE_FORCE" value="3" />
<option name="DOWHILE_BRACE_FORCE" value="3" />
<option name="WHILE_BRACE_FORCE" value="3" />
<option name="FOR_BRACE_FORCE" value="3" />
</codeStyleSettings>
</code_scheme>
</component>
\ No newline at end of file
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
</state>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="TsLint" enabled="true" level="WARNING" enabled_by_default="true" />
</profile>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="NodePackageJsonFileManager">
<packageJsonPaths />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/es-sink.iml" filepath="$PROJECT_DIR$/.idea/es-sink.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="958bfc8e-fe0a-49d4-894d-b6fd4975d8d9" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ComposerSettings">
<execution>
<executable />
</execution>
</component>
<component name="FileEditorManager">
<leaf>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/Readme.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="SPLIT">
<first_editor relative-caret-position="90">
<caret line="2" selection-start-line="2" selection-end-line="2" />
</first_editor>
<second_editor />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18">
<caret line="1" column="18" selection-start-line="1" selection-start-column="18" selection-end-line="1" selection-end-column="18" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/index.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="173">
<caret line="24" column="31" selection-start-line="24" selection-start-column="31" selection-end-line="24" selection-end-column="31" />
<folding>
<element signature="n#fmt#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/Dockerfile">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="450">
<caret line="25" column="13" selection-start-line="25" selection-start-column="13" selection-end-line="25" selection-end-column="13" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="288">
<caret line="16" column="7" lean-forward="true" selection-start-line="16" selection-start-column="7" selection-end-line="16" selection-end-column="7" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/connection.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="144">
<caret line="10" column="32" selection-start-line="10" selection-start-column="32" selection-end-line="10" selection-end-column="32" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/node_modules/elasticsearch/src/lib/apis/master.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="434">
<caret line="24" column="270" selection-start-line="24" selection-start-column="270" selection-end-line="24" selection-end-column="270" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/elasticsearch-transformer.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="90">
<caret line="5" column="37" selection-start-line="5" selection-start-column="13" selection-end-line="5" selection-end-column="37" />
<folding>
<element signature="e#0#42#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>bulk</find>
<find>promis</find>
</findStrings>
<dirStrings>
<dir>$PROJECT_DIR$/node_modules/elasticsearch/src/lib</dir>
</dirStrings>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/Readme.md" />
<option value="$PROJECT_DIR$/package.json" />
<option value="$PROJECT_DIR$/src/neo4j-stream-writer.ts" />
<option value="$PROJECT_DIR$/src/elasticsearch-transformer.ts" />
<option value="$PROJECT_DIR$/config.json" />
<option value="$PROJECT_DIR$/src/index.ts" />
<option value="$PROJECT_DIR$/Dockerfile" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="y" value="25" />
<option name="width" value="1890" />
<option name="height" value="1036" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="es-sink" type="b2602c69:ProjectViewProjectNode" />
<item name="es-sink" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="es-sink" type="b2602c69:ProjectViewProjectNode" />
<item name="es-sink" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="node.js.detected.package.tslint" value="true" />
<property name="node.js.path.for.package.tslint" value="project" />
<property name="node.js.selected.package.tslint" value="(autodetect)" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="nodejs_package_manager_path" value="npm" />
<property name="ts.external.directory.path" value="$PROJECT_DIR$/node_modules/typescript/lib" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/src" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="958bfc8e-fe0a-49d4-894d-b6fd4975d8d9" name="Default Changelist" comment="" />
<created>1558623705780</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1558623705780</updated>
<workItem from="1558623707064" duration="2914000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="2914000" />
</component>
<component name="ToolWindowManager">
<frame x="-1" y="25" width="1922" height="1056" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="npm" side_tool="true" />
<window_info id="Favorites" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="TypeScript" />
<window_info anchor="bottom" id="Docker" show_stripe_button="false" />
<window_info anchor="bottom" id="Database Changes" />
<window_info anchor="bottom" id="Version Control" />
<window_info anchor="bottom" id="Terminal" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/Readme.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="SPLIT">
<first_editor relative-caret-position="90">
<caret line="2" selection-start-line="2" selection-end-line="2" />
</first_editor>
<second_editor />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18">
<caret line="1" column="18" selection-start-line="1" selection-start-column="18" selection-end-line="1" selection-end-column="18" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/node_modules/elasticsearch/src/lib/apis/master.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="434">
<caret line="24" column="270" selection-start-line="24" selection-start-column="270" selection-end-line="24" selection-end-column="270" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/node_modules/@types/elasticsearch/index.d.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="252">
<caret line="26" column="4" selection-start-line="26" selection-start-column="4" selection-end-line="26" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/neo4j-stream-writer.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="126">
<caret line="7" column="24" selection-start-line="7" selection-start-column="24" selection-end-line="7" selection-end-column="24" />
<folding>
<element signature="e#246#278#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/elasticsearch-transformer.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="90">
<caret line="5" column="37" selection-start-line="5" selection-start-column="13" selection-end-line="5" selection-end-column="37" />
<folding>
<element signature="e#0#42#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="288">
<caret line="16" column="7" lean-forward="true" selection-start-line="16" selection-start-column="7" selection-end-line="16" selection-end-column="7" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/connection.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="144">
<caret line="10" column="32" selection-start-line="10" selection-start-column="32" selection-end-line="10" selection-end-column="32" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/index.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="173">
<caret line="24" column="31" selection-start-line="24" selection-start-column="31" selection-end-line="24" selection-end-column="31" />
<folding>
<element signature="n#fmt#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/Dockerfile">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="450">
<caret line="25" column="13" selection-start-line="25" selection-start-column="13" selection-end-line="25" selection-end-column="13" />
</state>
</provider>
</entry>
</component>
</project>
\ No newline at end of file
FROM node:10-slim
# add basic libs
RUN apt-get update && apt-get install -y \
bash \
g++ \
gcc \
ca-certificates \
make \
python \
bash \
git
# Create app directory
RUN mkdir -p /usr/local/app
# Move to the app directory
WORKDIR /usr/local/app
# copy app to the container
COPY package.json package-lock.json config.json tsconfig.json /usr/local/app/
# Install dependencies
RUN npm install
# build stuff
COPY src /usr/local/app/src
RUN npm run build
# run app
CMD node dist/index.js
# Es sink
After alotof struggles and undebuggable prolem wit kafka-connect we implemented our own elastsearch sink. Its a very simple kafka-consumer who forward documents to elasticsearch
{
"kafka": {
"broker": "localhost:29092",
"schema-registry": "http://localhost:8081"
},
"consumer": {
"config": {
"group.id": "es-sink",
"socket.keepalive.enable": true,
"enable.auto.commit": true,
"queued.max.messages.kbytes": 100
},
"topics": {
"topics": ["geolinker"],
"config": {
"auto.offset.reset": "largest"
}
},
"stream": {
"request.required.acks": 1
}
},
"neo4j": {
"config": {
"uri": "bolt://localhost:7687",
"user": "neo4j",
"password": "tester"
}
},
"log-dir": "../log/",
"reporter": {
"url": "http://dashboard-api"
},
"elasticsearch": {
"server": "http://localhost",
"port": 9200
}
}
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const elasticsearch = require("elasticsearch");
/**
* Connect to elastic-search
* @param nconf
* @returns {Client}
*/
function createClient(nconf) {
return new elasticsearch.Client({
hosts: [
`${nconf.get('elasticsearch:server')}:${nconf.get('elasticsearch:port')}/`,
],
});
}
exports.createClient = createClient;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29ubmVjdGlvbi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9jb25uZWN0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQUEsK0NBQWdEO0FBSWhEOzs7O0dBSUc7QUFDSCxzQkFBNkIsS0FBZTtJQUN4QyxPQUFPLElBQUksYUFBYSxDQUFDLE1BQU0sQ0FBQztRQUM1QixLQUFLLEVBQUU7WUFDSCxHQUFHLEtBQUssQ0FBQyxHQUFHLENBQUMsc0JBQXNCLENBQUMsSUFBSSxLQUFLLENBQUMsR0FBRyxDQUFDLG9CQUFvQixDQUFDLEdBQUc7U0FDN0U7S0FDSixDQUFDLENBQUM7QUFDUCxDQUFDO0FBTkQsb0NBTUMifQ==
\ No newline at end of file
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const connection_1 = require("./connection");
const stream_1 = require("stream");
class ElasticsearchTransformer extends stream_1.Transform {
constructor(options = {}, core) {
super(options);
/**
* Batch size
* @type {string}
*/
this.batchSize = 500;
/**
* Cache for batch
*/
this.batch = [];
/**
* Target index
*/
this.index = 'geolinker';
/**
* Timeout if something went wrong
*/
this.timeout = 500;
this.core = core;
this.client = connection_1.createClient(core.getNconf());
this.timeout = options.timeout || 500;
this.batchSize = options.batchSize || 500;
}
_transform(chunk, encoding, callback) {
return __awaiter(this, void 0, void 0, function* () {
this.batch.push(this.prepareBulk(chunk.parsed));
if (this.batch.length > this.batchSize) {
try {
yield this.client.bulk({ body: this.batch });
}
catch (err) {
console.error(err);
setTimeout(() => this._transform(chunk, encoding, callback), this.timeout);
}
this.batch = [];
}
setImmediate(callback);
});