Skip to content

Commit 05fc150

Browse files
author
Ferenc Csaky
committed
[FLINK-33557] Externalize Cassandra Python connector code
1 parent 1f171fe commit 05fc150

12 files changed

Lines changed: 1268 additions & 1 deletion

File tree

‎.gitignore‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,18 @@ tools/flink
3636
tools/flink-*
3737
tools/releasing/release
3838
tools/japicmp-output
39+
40+
# Generated files, do not store in git
41+
flink-python/apache_flink_connectors_cassandra.egg-info/
42+
flink-python/.tox/
43+
flink-python/build
44+
flink-python/dist
45+
flink-python/dev/download
46+
flink-python/dev/.conda/
47+
flink-python/dev/log/
48+
flink-python/dev/.stage.txt
49+
flink-python/dev/install_command.sh
50+
flink-python/dev/lint-python.sh
51+
flink-python/dev/build-wheels.sh
52+
flink-python/dev/glibc_version_fix.h
53+
flink-python/dev/dev-requirements.txt

‎flink-connector-cassandra/pom.xml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ under the License.
3737

3838
<!-- Allow users to pass custom connector versions -->
3939
<properties>
40-
<scala.binary.version>2.12</scala.binary.version>
4140
<scala-library.version>2.12.7</scala-library.version>
4241
<slf4j.version>1.7.36</slf4j.version>
4342
<log4j.version>2.17.1</log4j.version>

‎flink-python/MANIFEST.in‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
graft pyflink
20+
global-exclude *.py[cod] __pycache__ .DS_Store

‎flink-python/README.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
This is official Apache Flink Cassandra Python connector.
2+
3+
For the latest information about Flink connector, please visit our website at:
4+
5+
https://flink.apache.org
6+
7+
and our GitHub Account for Cassandra connector
8+
9+
https://github.com/apache/flink-connector-cassandra
10+
11+
If you have any questions, ask on our Mailing lists:
12+
13+
user@flink.apache.org
14+
dev@flink.apache.org

‎flink-python/dev/integration_test.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
################################################################################
19+
20+
function test_module() {
21+
module="$FLINK_PYTHON_DIR/pyflink/$1"
22+
echo "test module $module"
23+
pytest --durations=20 ${module} $2
24+
if [[ $? -ne 0 ]]; then
25+
echo "test module $module failed"
26+
exit 1
27+
fi
28+
}
29+
30+
function test_all_modules() {
31+
# test datastream module
32+
test_module "datastream"
33+
}
34+
35+
# CURRENT_DIR is "<root-project-folder>/flink-python/dev/"
36+
CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)"
37+
38+
# FLINK_PYTHON_DIR is "<root-project-folder>/flink-python/"
39+
FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR")
40+
41+
# set the FLINK_TEST_LIB_DIR to "<root-project-folder>/flink-connector-cassandra/flink-python-connector-cassandra/target/dep..."
42+
export FLINK_TEST_LIBS="${FLINK_PYTHON_DIR}/target/test-dependencies/*"
43+
44+
# Temporarily update the installed 'pyflink_gateway_server.py' files with the new one
45+
# Needed only until Flink 1.19 release
46+
echo "Checking ${FLINK_PYTHON_DIR} for 'pyflink_gateway_server.py'"
47+
find "${FLINK_PYTHON_DIR}/.tox" -name pyflink_gateway_server.py -exec cp "${FLINK_PYTHON_DIR}/pyflink/pyflink_gateway_server.py" {} \;
48+
49+
# python test
50+
test_all_modules

‎flink-python/pom.xml‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<parent>
27+
<groupId>org.apache.flink</groupId>
28+
<artifactId>flink-connector-cassandra-parent</artifactId>
29+
<version>4.0-SNAPSHOT</version>
30+
</parent>
31+
32+
<artifactId>flink-connector-cassandra-python</artifactId>
33+
<name>Flink : Connectors : Cassandra : Python</name>
34+
35+
<packaging>pom</packaging>
36+
37+
<dependencies>
38+
<dependency>
39+
<groupId>org.apache.flink</groupId>
40+
<artifactId>flink-connector-cassandra_${scala.binary.version}</artifactId>
41+
<version>${project.version}</version>
42+
<scope>provided</scope>
43+
<exclusions>
44+
<exclusion>
45+
<groupId>org.slf4j</groupId>
46+
<artifactId>slf4j-api</artifactId>
47+
</exclusion>
48+
</exclusions>
49+
</dependency>
50+
51+
<dependency>
52+
<groupId>org.apache.flink</groupId>
53+
<artifactId>flink-runtime</artifactId>
54+
<version>${flink.version}</version>
55+
<scope>test</scope>
56+
<type>test-jar</type>
57+
<exclusions>
58+
<exclusion>
59+
<groupId>org.slf4j</groupId>
60+
<artifactId>slf4j-api</artifactId>
61+
</exclusion>
62+
<exclusion>
63+
<groupId>com.esotericsoftware.kryo</groupId>
64+
<artifactId>kryo</artifactId>
65+
</exclusion>
66+
</exclusions>
67+
</dependency>
68+
69+
<dependency>
70+
<groupId>org.apache.flink</groupId>
71+
<artifactId>flink-test-utils</artifactId>
72+
<version>${flink.version}</version>
73+
<scope>test</scope>
74+
<exclusions>
75+
<exclusion>
76+
<groupId>org.slf4j</groupId>
77+
<artifactId>slf4j-api</artifactId>
78+
</exclusion>
79+
</exclusions>
80+
</dependency>
81+
82+
<dependency>
83+
<groupId>org.apache.flink</groupId>
84+
<artifactId>flink-connector-test-utils</artifactId>
85+
<version>${flink.version}</version>
86+
<scope>test</scope>
87+
<exclusions>
88+
<exclusion>
89+
<groupId>org.slf4j</groupId>
90+
<artifactId>slf4j-api</artifactId>
91+
</exclusion>
92+
<exclusion>
93+
<groupId>org.apache.commons</groupId>
94+
<artifactId>commons-compress</artifactId>
95+
</exclusion>
96+
</exclusions>
97+
</dependency>
98+
</dependencies>
99+
100+
<build>
101+
<plugins>
102+
<plugin>
103+
<groupId>org.apache.maven.plugins</groupId>
104+
<artifactId>maven-antrun-plugin</artifactId>
105+
<executions>
106+
<execution>
107+
<id>clean</id>
108+
<phase>clean</phase>
109+
<goals>
110+
<goal>run</goal>
111+
</goals>
112+
<configuration>
113+
<target>
114+
<delete dir="${project.basedir}/.tox"/>
115+
<delete
116+
dir="${project.basedir}/apache_flink_connectors_cassandra.egg-info"/>
117+
<delete dir="${project.basedir}/dev/.conda"/>
118+
<delete dir="${project.basedir}/dev/download"/>
119+
<delete dir="${project.basedir}/dev/log"/>
120+
<delete dir="${project.basedir}/build"/>
121+
<delete dir="${project.basedir}/dist"/>
122+
<delete dir="${project.basedir}/pyflink/lib"/>
123+
<delete file="${project.basedir}/dev/.stage.txt"/>
124+
<delete file="${project.basedir}/dev/install_command.sh"/>
125+
<delete file="${project.basedir}/dev/lint-python.sh"/>
126+
<delete file="${project.basedir}/dev/build-wheels.sh"/>
127+
<delete file="${project.basedir}/dev/glibc_version_fix.h"/>
128+
<delete file="${project.basedir}/dev/dev-requirements.txt"/>
129+
<delete
130+
file="${project.basedir}/pyflink/datastream/connectors/cassandra_connector_version.py"/>
131+
</target>
132+
</configuration>
133+
</execution>
134+
</executions>
135+
</plugin>
136+
<plugin>
137+
<groupId>org.apache.maven.plugins</groupId>
138+
<artifactId>maven-dependency-plugin</artifactId>
139+
<executions>
140+
<execution>
141+
<phase>package</phase>
142+
<goals>
143+
<goal>copy</goal>
144+
</goals>
145+
<configuration>
146+
<artifactItems>
147+
<artifactItem>
148+
<groupId>org.apache.flink</groupId>
149+
<artifactId>flink-connector-cassandra_${scala.binary.version}</artifactId>
150+
</artifactItem>
151+
<artifactItem>
152+
<groupId>org.apache.flink</groupId>
153+
<artifactId>flink-runtime</artifactId>
154+
<!-- Don't use test-jar type because of a bug in the plugin (MDEP-587). -->
155+
<classifier>tests</classifier>
156+
</artifactItem>
157+
<artifactItem>
158+
<groupId>org.apache.flink</groupId>
159+
<artifactId>flink-test-utils</artifactId>
160+
</artifactItem>
161+
<artifactItem>
162+
<groupId>org.apache.flink</groupId>
163+
<artifactId>flink-connector-test-utils</artifactId>
164+
</artifactItem>
165+
</artifactItems>
166+
<outputDirectory>${project.build.directory}/test-dependencies</outputDirectory>
167+
</configuration>
168+
</execution>
169+
<execution>
170+
<id>copy-dependencies</id>
171+
<phase>package</phase>
172+
<goals>
173+
<goal>copy-dependencies</goal>
174+
</goals>
175+
<configuration>
176+
<includeGroupIds>junit</includeGroupIds>
177+
<outputDirectory>${project.build.directory}/test-dependencies</outputDirectory>
178+
</configuration>
179+
</execution>
180+
</executions>
181+
</plugin>
182+
<!-- Download the testing infra sources from the Flink main repository -->
183+
<plugin>
184+
<groupId>org.codehaus.mojo</groupId>
185+
<artifactId>wagon-maven-plugin</artifactId>
186+
<version>2.0.2</version>
187+
<executions>
188+
<execution>
189+
<id>download-install</id>
190+
<phase>validate</phase>
191+
<goals>
192+
<goal>download-single</goal>
193+
</goals>
194+
<configuration>
195+
<url>
196+
https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/install_command.sh
197+
</url>
198+
<toDir>${project.basedir}/dev</toDir>
199+
<skip>${python.infra.download.skip}</skip>
200+
</configuration>
201+
</execution>
202+
<execution>
203+
<id>download-lint</id>
204+
<phase>validate</phase>
205+
<goals>
206+
<goal>download-single</goal>
207+
</goals>
208+
<configuration>
209+
<url>
210+
https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/lint-python.sh
211+
</url>
212+
<toDir>${project.basedir}/dev</toDir>
213+
<skip>${python.infra.download.skip}</skip>
214+
</configuration>
215+
</execution>
216+
<execution>
217+
<id>download-build-wheels</id>
218+
<phase>validate</phase>
219+
<goals>
220+
<goal>download-single</goal>
221+
</goals>
222+
<configuration>
223+
<url>
224+
https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/build-wheels.sh
225+
</url>
226+
<toDir>${project.basedir}/dev</toDir>
227+
<skip>${python.infra.download.skip}</skip>
228+
</configuration>
229+
</execution>
230+
<execution>
231+
<id>download-build-version-header</id>
232+
<phase>validate</phase>
233+
<goals>
234+
<goal>download-single</goal>
235+
</goals>
236+
<configuration>
237+
<url>
238+
https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/glibc_version_fix.h
239+
</url>
240+
<toDir>${project.basedir}/dev</toDir>
241+
<skip>${python.infra.download.skip}</skip>
242+
</configuration>
243+
</execution>
244+
</executions>
245+
</plugin>
246+
</plugins>
247+
</build>
248+
</project>

0 commit comments

Comments
 (0)