Skip to content

Commit 4076f9d

Browse files
mzitnikmzitnik
authored
Merge pull request #38 from ClickHouse/integration-tests
Integration tests Cluster
2 parents ea762f0 + ce821a5 commit 4076f9d

File tree

24 files changed

+1180
-194
lines changed

24 files changed

+1180
-194
lines changed

‎.github/workflows/integration-tests.yaml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Integration Tests CI (Java)
2+
3+
on:
4+
push:
5+
paths-ignore:
6+
- '**.md'
7+
- 'LICENSE'
8+
- 'examples'
9+
10+
jobs:
11+
test-integration-tests:
12+
runs-on: ubuntu-latest
13+
timeout-minutes: 10
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
clickhouse: [ "23.7", "24.3", "latest", "cloud" ]
18+
flink: [ "1.17.2", "1.18.1", "1.19.3", "1.20.2", "2.0.0", "latest"]
19+
steps:
20+
- name: Check for Cloud Credentials
21+
id: check-cloud-credentials
22+
run: |
23+
if [[ "${{ matrix.clickhouse }}" == "cloud" && (-z "${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}" || -z "${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}") ]]; then
24+
echo "SKIP_STEP=true" >> $GITHUB_ENV
25+
else
26+
echo "SKIP_STEP=false" >> $GITHUB_ENV
27+
fi
28+
shell: bash
29+
- uses: actions/checkout@v3
30+
if: env.SKIP_STEP != 'true'
31+
- name: Set up JDK 17
32+
if: env.SKIP_STEP != 'true'
33+
uses: actions/setup-java@v3
34+
with:
35+
java-version: '17'
36+
distribution: 'adopt'
37+
architecture: x64
38+
- name: Publish locally flink-connector-clickhouse-1.17
39+
if: env.SKIP_STEP != 'true'
40+
uses: gradle/gradle-build-action@v2
41+
with:
42+
arguments: :flink-connector-clickhouse-1.17:publishToMavenLocal
43+
- name: Publish locally flink-connector-clickhouse-2.0.0
44+
if: env.SKIP_STEP != 'true'
45+
uses: gradle/gradle-build-action@v2
46+
with:
47+
arguments: :flink-connector-clickhouse-2.0.0:publishToMavenLocal
48+
- name: Generate Flink Covid App example 2.X
49+
if: env.SKIP_STEP != 'true'
50+
working-directory: ./examples/maven/flink-v2/covid
51+
run: mvn clean install
52+
- name: Generate Flink Covid App example 1.17+
53+
if: env.SKIP_STEP != 'true'
54+
working-directory: ./examples/maven/flink-v1.7/covid
55+
run: mvn clean install
56+
- name: Setup and execute Gradle 'integration-test' task
57+
if: env.SKIP_STEP != 'true'
58+
uses: gradle/gradle-build-action@v2
59+
env:
60+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
61+
CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}
62+
CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}
63+
FLINK_VERSION: ${{ matrix.flink }}
64+
with:
65+
arguments: :flink-connector-clickhouse-integration:test
66+
67+
68+
69+

‎.github/workflows/tests-scala.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on: [push]
55
jobs:
66
test-flink-17:
77
runs-on: ubuntu-latest
8+
timeout-minutes: 10
89
strategy:
910
fail-fast: false
1011
matrix:
@@ -42,6 +43,7 @@ jobs:
4243
arguments: :flink-connector-clickhouse-1.17:runScalaTests
4344
test-flink-2:
4445
runs-on: ubuntu-latest
46+
timeout-minutes: 10
4547
strategy:
4648
fail-fast: false
4749
matrix:

‎.github/workflows/tests.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on: [push]
55
jobs:
66
test-flink-17:
77
runs-on: ubuntu-latest
8+
timeout-minutes: 10
89
strategy:
910
fail-fast: false
1011
matrix:
@@ -23,11 +24,11 @@ jobs:
2324
shell: bash
2425
- uses: actions/checkout@v3
2526
if: env.SKIP_STEP != 'true'
26-
- name: Set up JDK 21
27+
- name: Set up JDK 17
2728
if: env.SKIP_STEP != 'true'
2829
uses: actions/setup-java@v3
2930
with:
30-
java-version: '21'
31+
java-version: '17'
3132
distribution: 'adopt'
3233
architecture: x64
3334
- name: Setup and execute Gradle 'test' task
@@ -42,6 +43,7 @@ jobs:
4243
arguments: :flink-connector-clickhouse-1.17:test
4344
test-flink-2:
4445
runs-on: ubuntu-latest
46+
timeout-minutes: 10
4547
strategy:
4648
fail-fast: false
4749
matrix:
@@ -59,11 +61,11 @@ jobs:
5961
shell: bash
6062
- uses: actions/checkout@v3
6163
if: env.SKIP_STEP != 'true'
62-
- name: Set up JDK 21
64+
- name: Set up JDK 17
6365
if: env.SKIP_STEP != 'true'
6466
uses: actions/setup-java@v3
6567
with:
66-
java-version: '21'
68+
java-version: '17'
6769
distribution: 'adopt'
6870
architecture: x64
6971
- name: Setup and execute Gradle 'test' task

‎build.gradle.kts‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@ plugins {
33
java
44
signing
55
id("com.gradleup.nmcp") version "0.0.8"
6-
id("com.github.johnrengelman.shadow") version "8.1.1"
7-
6+
id("com.gradleup.shadow") version "9.0.2"
87
}
98

109
val sinkVersion by extra("0.0.1")
1110
val flinkVersion by extra("1.18.0")
12-
val clickhouseVersion by extra("0.4.6")
11+
val clickhouseVersion by extra("0.9.1")
1312
val junitVersion by extra("5.8.2")
1413

1514
allprojects {
@@ -75,23 +74,4 @@ subprojects {
7574
"-s", "org.apache.flink.connector.clickhouse.test.scala.ClickHouseSinkTests"
7675
)
7776
}
78-
}
79-
80-
//sourceSets {
81-
// main {
82-
// scala {
83-
// srcDirs("src/main/scala")
84-
// }
85-
// java {
86-
// srcDirs("src/main/java")
87-
// }
88-
// }
89-
// test {
90-
// scala {
91-
// srcDirs("src/test/scala")
92-
// }
93-
// java {
94-
// srcDirs("src/test/java")
95-
// }
96-
// }
97-
//}
77+
}

‎examples/maven/flink-v1.7/covid/pom.xml‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<groupId>com.clickhouse.example.covid</groupId>
24+
<artifactId>covid</artifactId>
25+
<version>1.0-SNAPSHOT</version>
26+
<packaging>jar</packaging>
27+
28+
<name>Flink Quickstart Job</name>
29+
30+
<properties>
31+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
32+
<flink.version>1.17.2</flink.version>
33+
<target.java.version>11</target.java.version>
34+
<scala.binary.version>2.12</scala.binary.version>
35+
<maven.compiler.source>${target.java.version}</maven.compiler.source>
36+
<maven.compiler.target>${target.java.version}</maven.compiler.target>
37+
<log4j.version>2.24.1</log4j.version>
38+
</properties>
39+
40+
<repositories>
41+
42+
<repository>
43+
<id>apache.snapshots</id>
44+
<name>Apache Development Snapshot Repository</name>
45+
<url>https://repository.apache.org/content/repositories/snapshots/</url>
46+
<releases>
47+
<enabled>false</enabled>
48+
</releases>
49+
<snapshots>
50+
<enabled>true</enabled>
51+
</snapshots>
52+
</repository>
53+
</repositories>
54+
55+
<dependencies>
56+
<!-- Apache Flink dependencies -->
57+
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
58+
<dependency>
59+
<groupId>org.apache.flink</groupId>
60+
<artifactId>flink-streaming-java</artifactId>
61+
<version>${flink.version}</version>
62+
<scope>provided</scope>
63+
</dependency>
64+
<dependency>
65+
<groupId>org.apache.flink</groupId>
66+
<artifactId>flink-clients</artifactId>
67+
<version>${flink.version}</version>
68+
<scope>provided</scope>
69+
</dependency>
70+
<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-files -->
71+
<dependency>
72+
<groupId>org.apache.flink</groupId>
73+
<artifactId>flink-connector-files</artifactId>
74+
<version>${flink.version}</version>
75+
<scope>provided</scope>
76+
</dependency>
77+
78+
<dependency>
79+
<groupId>com.clickhouse.flink</groupId>
80+
<artifactId>flink-connector-clickhouse-1.17</artifactId>
81+
<version>0.0.1</version>
82+
<classifier>all</classifier>
83+
</dependency>
84+
85+
86+
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
87+
88+
<!-- Example:
89+
90+
<dependency>
91+
<groupId>org.apache.flink</groupId>
92+
<artifactId>flink-connector-kafka</artifactId>
93+
<version>3.0.0-1.17</version>
94+
</dependency>
95+
-->
96+
97+
<!-- Add logging framework, to produce console output when running in the IDE. -->
98+
<!-- These dependencies are excluded from the application JAR by default. -->
99+
<dependency>
100+
<groupId>org.apache.logging.log4j</groupId>
101+
<artifactId>log4j-slf4j-impl</artifactId>
102+
<version>${log4j.version}</version>
103+
<scope>runtime</scope>
104+
</dependency>
105+
<dependency>
106+
<groupId>org.apache.logging.log4j</groupId>
107+
<artifactId>log4j-api</artifactId>
108+
<version>${log4j.version}</version>
109+
<scope>runtime</scope>
110+
</dependency>
111+
<dependency>
112+
<groupId>org.apache.logging.log4j</groupId>
113+
<artifactId>log4j-core</artifactId>
114+
<version>${log4j.version}</version>
115+
<scope>runtime</scope>
116+
</dependency>
117+
</dependencies>
118+
119+
<build>
120+
<plugins>
121+
122+
<!-- Java Compiler -->
123+
<plugin>
124+
<groupId>org.apache.maven.plugins</groupId>
125+
<artifactId>maven-compiler-plugin</artifactId>
126+
<version>3.1</version>
127+
<configuration>
128+
<source>${target.java.version}</source>
129+
<target>${target.java.version}</target>
130+
</configuration>
131+
</plugin>
132+
133+
<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
134+
<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
135+
<plugin>
136+
<groupId>org.apache.maven.plugins</groupId>
137+
<artifactId>maven-shade-plugin</artifactId>
138+
<version>3.1.1</version>
139+
<executions>
140+
<!-- Run shade goal on package phase -->
141+
<execution>
142+
<phase>package</phase>
143+
<goals>
144+
<goal>shade</goal>
145+
</goals>
146+
<configuration>
147+
<createDependencyReducedPom>false</createDependencyReducedPom>
148+
<artifactSet>
149+
<excludes>
150+
<exclude>org.apache.flink:flink-shaded-force-shading</exclude>
151+
<exclude>com.google.code.findbugs:jsr305</exclude>
152+
<exclude>org.slf4j:*</exclude>
153+
<exclude>org.apache.logging.log4j:*</exclude>
154+
</excludes>
155+
</artifactSet>
156+
<filters>
157+
<filter>
158+
<!-- Do not copy the signatures in the META-INF folder.
159+
Otherwise, this might cause SecurityExceptions when using the JAR. -->
160+
<artifact>*:*</artifact>
161+
<excludes>
162+
<exclude>META-INF/*.SF</exclude>
163+
<exclude>META-INF/*.DSA</exclude>
164+
<exclude>META-INF/*.RSA</exclude>
165+
</excludes>
166+
</filter>
167+
</filters>
168+
<transformers>
169+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
170+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
171+
<mainClass>com.clickhouse.example.covid.DataStreamJob</mainClass>
172+
</transformer>
173+
</transformers>
174+
</configuration>
175+
</execution>
176+
</executions>
177+
</plugin>
178+
</plugins>
179+
180+
<pluginManagement>
181+
<plugins>
182+
183+
<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
184+
<plugin>
185+
<groupId>org.eclipse.m2e</groupId>
186+
<artifactId>lifecycle-mapping</artifactId>
187+
<version>1.0.0</version>
188+
<configuration>
189+
<lifecycleMappingMetadata>
190+
<pluginExecutions>
191+
<pluginExecution>
192+
<pluginExecutionFilter>
193+
<groupId>org.apache.maven.plugins</groupId>
194+
<artifactId>maven-shade-plugin</artifactId>
195+
<versionRange>[3.1.1,)</versionRange>
196+
<goals>
197+
<goal>shade</goal>
198+
</goals>
199+
</pluginExecutionFilter>
200+
<action>
201+
<ignore/>
202+
</action>
203+
</pluginExecution>
204+
<pluginExecution>
205+
<pluginExecutionFilter>
206+
<groupId>org.apache.maven.plugins</groupId>
207+
<artifactId>maven-compiler-plugin</artifactId>
208+
<versionRange>[3.1,)</versionRange>
209+
<goals>
210+
<goal>testCompile</goal>
211+
<goal>compile</goal>
212+
</goals>
213+
</pluginExecutionFilter>
214+
<action>
215+
<ignore/>
216+
</action>
217+
</pluginExecution>
218+
</pluginExecutions>
219+
</lifecycleMappingMetadata>
220+
</configuration>
221+
</plugin>
222+
</plugins>
223+
</pluginManagement>
224+
</build>
225+
</project>

0 commit comments

Comments
 (0)