Skip to content

Commit 588d310

Browse files
mmolimarmmolimar
authored
Merge pull request #52 from mmolimar/develop
Release version 1.0.0
2 parents cd91962 + 175df2a commit 588d310

File tree

96 files changed

+5371
-3364
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+5371
-3364
lines changed

‎.travis.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1+
dist: trusty
12
language: java
23
jdk:
34
- oraclejdk8
4-
sudo: false
55
install:
6-
- mvn test-compile -DskipTests=true -Dmaven.javadoc.skip=true -B -V
6+
- mvn test-compile -DskipTests=true -Dmaven.javadoc.skip=true -B -V
77
script:
8-
- mvn test jacoco:report
8+
- mvn test jacoco:report
99
after_success:
10-
- mvn coveralls:report
10+
- mvn coveralls:report

‎Dockerfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM confluentinc/cp-kafka-connect-base:5.5.0
2+
3+
ARG PROJECT_VERSION
4+
ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"
5+
6+
COPY ./target/components/packages/mmolimar-kafka-connect-fs-${PROJECT_VERSION}.zip /tmp/kafka-connect-fs.zip
7+
RUN confluent-hub install --no-prompt /tmp/kafka-connect-fs.zip && rm -rf /tmp/kafka-connect-fs.zip

‎LICENSE

+1-1
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@
187187
same "printed page" as the copyright notice for easier
188188
identification within third-party archives.
189189

190-
Copyright [yyyy] [name of copyright owner]
190+
Copyright 2017 Mario Molina
191191

192192
Licensed under the Apache License, Version 2.0 (the "License");
193193
you may not use this file except in compliance with the License.

‎README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Kafka Connect FileSystem Connector [![Build Status](https://travis-ci.org/mmolimar/kafka-connect-fs.svg?branch=master)](https://travis-ci.org/mmolimar/kafka-connect-fs)[![Coverage Status](https://coveralls.io/repos/github/mmolimar/kafka-connect-fs/badge.svg?branch=master)](https://coveralls.io/github/mmolimar/kafka-connect-fs?branch=master)
22

3-
**kafka-connect-fs** is a [Kafka Connector](http://kafka.apache.org/documentation.html#connect)
3+
**kafka-connect-fs** is a [Kafka Connector](https://kafka.apache.org/documentation.html#connect)
44
for reading records from files in the file systems specified and load them into Kafka.
55

6-
Documentation for this connector can be found [here](http://kafka-connect-fs.readthedocs.io/).
6+
Documentation for this connector can be found [here](https://kafka-connect-fs.readthedocs.io/).
77

88
## Development
99

@@ -13,7 +13,7 @@ kafka-connect-fs with Maven using the standard lifecycle phases.
1313
## FAQ
1414

1515
Some frequently asked questions on Kafka Connect FileSystem Connector can be found here -
16-
http://kafka-connect-fs.readthedocs.io/en/latest/faq.html
16+
https://kafka-connect-fs.readthedocs.io/en/latest/faq.html
1717

1818
## Contribute
1919

‎config/kafka-connect-fs.properties

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
name=FsSourceConnector
22
connector.class=com.github.mmolimar.kafka.connect.fs.FsSourceConnector
33
tasks.max=1
4-
fs.uris=file:///data,hdfs://localhost:9000/
4+
fs.uris=file:///data,hdfs://localhost:8020/data
55
topic=mytopic
66
policy.class=com.github.mmolimar.kafka.connect.fs.policy.SimplePolicy
77
policy.recursive=true
8-
policy.regexp=^[0-9]*\.txt$
8+
policy.regexp=^.*\.txt$
99
file_reader.class=com.github.mmolimar.kafka.connect.fs.file.reader.TextFileReader

‎docker-compose.yml

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
version: '3'
2+
services:
3+
cp-zookeeper:
4+
image: confluentinc/cp-zookeeper:5.5.0
5+
hostname: zookeeper
6+
container_name: zookeeper
7+
ports:
8+
- "2181:2181"
9+
environment:
10+
ZOOKEEPER_CLIENT_PORT: 2181
11+
ZOOKEEPER_TICK_TIME: 2000
12+
13+
cp-kafka:
14+
image: confluentinc/cp-kafka:5.5.0
15+
hostname: kafka
16+
container_name: kafka
17+
depends_on:
18+
- cp-zookeeper
19+
ports:
20+
- "29092:29092"
21+
- "9092:9092"
22+
environment:
23+
KAFKA_BROKER_ID: 1
24+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
25+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
26+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
27+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
28+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
29+
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
30+
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
31+
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
32+
CONFLUENT_METRICS_ENABLE: 'false'
33+
34+
cp-schema-registry:
35+
image: confluentinc/cp-schema-registry:5.5.0
36+
hostname: schema-registry
37+
container_name: schema-registry
38+
depends_on:
39+
- cp-zookeeper
40+
- cp-kafka
41+
ports:
42+
- "8081:8081"
43+
environment:
44+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
45+
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
46+
47+
connect-fs:
48+
image: mmolimar/kafka-connect-fs:1.0.0
49+
container_name: connect
50+
depends_on:
51+
- cp-kafka
52+
- cp-schema-registry
53+
ports:
54+
- "8083:8083"
55+
- "8000:8000"
56+
environment:
57+
CONNECT_BOOTSTRAP_SERVERS: 'kafka:29092'
58+
CONNECT_REST_ADVERTISED_HOST_NAME: connect
59+
CONNECT_REST_PORT: 8083
60+
CONNECT_GROUP_ID: compose-connect-group
61+
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
62+
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
63+
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
64+
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
65+
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
66+
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
67+
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
68+
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
69+
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
70+
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
71+
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
72+
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
73+
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
74+
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components/"
75+
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
76+
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
77+
KAFKA_OPTS: "-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n"

‎docs/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@ help:
1717
# Catch-all target: route all unknown targets to Sphinx using the new
1818
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
1919
%: Makefile
20-
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
20+
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

‎docs/source/conf.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@
5555
# built documents.
5656
#
5757
# The short X.Y version.
58-
version = '0.1'
58+
version = '1.0'
5959
# The full version, including alpha/beta/rc tags.
60-
release = '0.1'
60+
release = '1.0'
6161

6262
# The language for content autogenerated by Sphinx. Refer to documentation
6363
# for a list of supported languages.

0 commit comments

Comments
 (0)