Skip to content

Commit b021e9c

Browse files
Will-Loautumnust
Will-Lo
authored and
autumnust
committed
[GOBBLIN-1365] Adds Github Action workflow for tests
Closes apache#3206 from Will-Lo/migrate-to-github- actions
1 parent f4c7513 commit b021e9c

File tree

25 files changed

+436
-28
lines changed

25 files changed

+436
-28
lines changed

‎.github/workflows/build_and_test.yaml

+164
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
name: Build and Run Tests
19+
20+
on:
21+
push:
22+
# Publish only on `master`
23+
branches:
24+
- master
25+
pull_request:
26+
branches:
27+
- master
28+
release:
29+
types: [published, edited]
30+
31+
jobs:
32+
build:
33+
name: Build repository
34+
runs-on: ubuntu-latest
35+
steps:
36+
- name: Check out the repo
37+
uses: actions/checkout@v2
38+
- name: Set up JDK 1.8
39+
uses: actions/setup-java@v1
40+
with:
41+
java-version: 1.8
42+
# Stores external dependencies, can be further improved with Gradle 6.1
43+
- name: Cache Gradle Dependencies
44+
uses: actions/cache@v2
45+
with:
46+
path: |
47+
~/.gradle/caches
48+
~/.gradle/wrapper
49+
# Only rebuild cache if build.gradle is changed
50+
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
51+
restore-keys: ${{ runner.os }}-gradle
52+
- name: Build repository
53+
run: |
54+
./gradlew --no-daemon clean build -x test -x javadoc -x findbugsMain -x findbugsTest -x checkstyleMain -x checkstyleJmh -x checkstyleTest -Dorg.gradle.parallel=true
55+
56+
test_coverage:
57+
runs-on: ubuntu-latest
58+
name: Generate test coverage
59+
needs: build
60+
steps:
61+
- name: Check out the repo
62+
uses: actions/checkout@v2
63+
- name: Set up JDK 1.8
64+
uses: actions/setup-java@v1
65+
with:
66+
java-version: 1.8
67+
- name: Cache Gradle Dependencies
68+
uses: actions/cache@v2
69+
with:
70+
path: |
71+
~/.gradle/caches
72+
~/.gradle/wrapper
73+
# Only rebuild cache if build.gradle is changed
74+
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
75+
restore-keys: ${{ runner.os }}-gradle
76+
- name: Generate code coverage
77+
run: |
78+
./gradlew -PskipTestGroup=disabledOnCI -Dorg.gradle.parallel=false -DjacocoBuild=1 $GOBBLIN_GRADLE_OPTS jacocoTestCoverage
79+
80+
static_checks:
81+
name: Run static checks
82+
runs-on: ubuntu-latest
83+
needs: build
84+
steps:
85+
- name: Check out the repo
86+
uses: actions/checkout@v2
87+
- name: Set up JDK 1.8
88+
uses: actions/setup-java@v1
89+
with:
90+
java-version: 1.8
91+
# Stores external dependencies, can be further improved with Gradle 6.1
92+
- name: Cache Gradle Dependencies
93+
uses: actions/cache@v2
94+
with:
95+
path: |
96+
~/.gradle/caches
97+
~/.gradle/wrapper
98+
# Only rebuild cache if build.gradle is changed
99+
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
100+
restore-keys: ${{ runner.os }}-gradle
101+
- name: Run CheckStyle and FindBugs
102+
run: |
103+
./gradlew --no-daemon -x javadoc findbugsMain checkstyleMain checkstyleTest checkstyleJmh
104+
105+
run_tests:
106+
timeout-minutes: 60
107+
env:
108+
GOBBLIN_GRADLE_OPTS: "--no-daemon -Dgobblin.metastore.testing.embeddedMysqlEnabled=false -PusePreinstalledMysql=true"
109+
strategy:
110+
matrix:
111+
test-group: ["Core Tests", "Service Tests", "Module Tests", "Other Tests"]
112+
fail-fast: false
113+
runs-on: ubuntu-latest
114+
needs: build
115+
services:
116+
mysql:
117+
image: mysql:5.7.32
118+
env:
119+
MYSQL_USER: testUser
120+
MYSQL_PASSWORD: testPassword
121+
MYSQL_DATABASE: test
122+
MYSQL_ROOT_PASSWORD: password
123+
ports:
124+
- 3306:3306
125+
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
126+
steps:
127+
- name: Check out the repo
128+
uses: actions/checkout@v2
129+
- name: Set up JDK 1.8
130+
uses: actions/setup-java@v1
131+
with:
132+
java-version: 1.8
133+
- name: Verify mysql connection
134+
run: |
135+
sudo apt-get install -y mysql-client
136+
mysql --host 127.0.0.1 --port 3306 -uroot -ppassword -e "SHOW DATABASES"
137+
- name: Cache Gradle Dependencies
138+
uses: actions/cache@v2
139+
with:
140+
path: |
141+
~/.gradle/caches
142+
~/.gradle/wrapper
143+
# Only rebuild cache if build.gradle is changed
144+
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
145+
restore-keys: ${{ runner.os }}-gradle
146+
- name: Run test group ${{ matrix.test-group }}
147+
# Write retry logic as integration tests can fail due to timing out or network problems
148+
run: |
149+
./gradlew getGroupedTests -PgroupName="${{matrix.test-group}}" > temp.txt
150+
TASKS=$(sed -n 's/CI Task: //p' temp.txt)
151+
echo $TASKS
152+
153+
n=0
154+
until [ "$n" -ge 3 ]
155+
do
156+
./gradlew -PskipTestGroup=disabledOnCI $GOBBLIN_GRADLE_OPTS $TASKS -Dorg.gradle.parallel=false && break
157+
n=$((n+1))
158+
if [[ $n -lt 3 ]]; then
159+
echo "Tests failed, retry attempt number $n"
160+
else
161+
exit 1
162+
fi
163+
sleep 10
164+
done

‎.travis.yml

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ stages:
3434

3535
before_script:
3636
- mysql -uroot -e "create user testUser identified by 'testPassword';"
37+
- mysql -uroot -e "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('password')"
38+
3739
script:
3840
- travis_retry ./travis/test.sh
3941
- travis_retry ./gradlew jacocoTestReport

‎build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ buildscript {
2222
apply from: 'gradle/scripts/repositories.gradle'
2323
apply from: 'gradle/scripts/defaultBuildProperties.gradle'
2424
apply from: 'gradle/scripts/computeVersions.gradle'
25+
apply from: 'gradle/scripts/ci-support.gradle'
2526

2627
apply from: file('gradle/scripts/buildscript.gradle'), to: buildscript
2728

‎gobblin-api/src/test/java/org/apache/gobblin/password/PasswordManagerTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333

3434
import com.google.common.io.Files;
3535

36-
@Test(enabled=false, groups = {"disabledOnTravis"} )
36+
@Test(enabled=false, groups = {"disabledOnCI"} )
3737
public class PasswordManagerTest {
3838

3939
@Test (enabled=false)

‎gobblin-cluster/src/test/java/org/apache/gobblin/cluster/ClusterIntegrationTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public boolean apply(Void input) {
146146
* We confirm the execution by again inspecting the zNode and ensuring its TargetState is START. </li>
147147
* </ul>
148148
*/
149-
@Test (enabled = false, dependsOnMethods = { "testJobShouldGetCancelled" }, groups = {"disabledOnTravis"})
149+
@Test (enabled = false, dependsOnMethods = { "testJobShouldGetCancelled" }, groups = {"disabledOnCI"})
150150
public void testJobRestartViaSpec() throws Exception {
151151
Config jobConfigOverrides = ClusterIntegrationTestUtils.buildSleepingJob(IntegrationJobCancelSuite.JOB_ID,
152152
IntegrationJobCancelSuite.TASK_STATE_FILE);

‎gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinClusterKillTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
*/
6060
// The kill tests are unreliable on Travis
6161
// Disabled GobblinClusterKillTest until reliability improves
62-
@Test(enabled=false, groups = {"disabledOnTravis"}, singleThreaded = true)
62+
@Test(enabled=false, groups = {"disabledOnCI"}, singleThreaded = true)
6363
public class GobblinClusterKillTest {
6464
public final static Logger LOG = LoggerFactory.getLogger(GobblinClusterKillTest.class);
6565
public static final String CLASS_NAME_BASED_PATH = "org/apache/gobblin/util/test/HelloWorldSource";
@@ -227,7 +227,7 @@ public boolean apply(Void input) {
227227

228228
// The kill tests are unreliable on Travis
229229
// Disabled GobblinClusterKillTest until reliability improves
230-
// @Test(groups = { "disabledOnTravis" }, dependsOnMethods = "testKillWorker")
230+
// @Test(groups = { "disabledOnCI" }, dependsOnMethods = "testKillWorker")
231231
public void testKillManager() throws IOException, TimeoutException, InterruptedException {
232232
// kill a manager to cause leader election. New leader will schedule a new job.
233233
_clusterManagers[0].disconnectHelixManager();
@@ -268,7 +268,7 @@ public boolean apply(Void input) {
268268

269269
// The kill tests are unreliable on Travis
270270
// Disabled GobblinClusterKillTest until reliability improves
271-
// @Test(groups = { "disabledOnTravis" }, enabled=true, dependsOnMethods = "testKillManager")
271+
// @Test(groups = { "disabledOnCI" }, enabled=true, dependsOnMethods = "testKillManager")
272272
public void testRestartManager() throws IOException, TimeoutException, InterruptedException {
273273
_clusterManagers[0].disconnectHelixManager();
274274

‎gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinTaskRunnerTest.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ public void testConnectHelixManagerWithRetry() {
204204
Assert.assertTrue(true);
205205
}
206206

207-
@Test (groups = {"disabledOnTravis"})
207+
@Test (groups = {"disabledOnCI"})
208208
public void testTaskAssignmentAfterHelixConnectionRetry()
209209
throws Exception {
210210
Config jobConfigOverrides = ClusterIntegrationTestUtils.buildSleepingJob(JOB_ID, TASK_STATE_FILE);
@@ -230,7 +230,7 @@ public void testTaskAssignmentAfterHelixConnectionRetry()
230230
helixManager.disconnect();
231231
}
232232

233-
@Test (groups = {"disabledOnTravis"}, dependsOnMethods = "testSendReceiveShutdownMessage", expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*ContainerHealthCheckException.*")
233+
@Test (groups = {"disabledOnCI"}, dependsOnMethods = "testSendReceiveShutdownMessage", expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*ContainerHealthCheckException.*")
234234
public void testShutdownOnHealthCheckFailure() throws Exception {
235235
this.gobblinTaskRunnerHealthCheck.connectHelixManager();
236236

‎gobblin-cluster/src/test/java/org/apache/gobblin/cluster/HelixAssignedParticipantCheckTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public void setUp()
5656
InstanceType.SPECTATOR, zkConnectString);
5757
}
5858

59-
@Test (groups = {"disabledOnTravis"})
59+
@Test (groups = {"disabledOnCI"})
6060
//Test disabled on Travis because cluster integration tests are generally flaky on Travis.
6161
public void testExecute() throws Exception {
6262
suite.startCluster();

‎gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/converter/DecryptConverterTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
/**
4949
* Unit tests for {@link DecryptConverter}.
5050
*/
51-
@Test(enabled=false, groups = { "gobblin.data.management.copy.converter", "disabledOnTravis" })
51+
@Test(enabled=false, groups = { "gobblin.data.management.copy.converter", "disabledOnCI" })
5252
public class DecryptConverterTest {
5353

5454
private final File masterPwdFile = new File("masterPwd");

‎gobblin-metastore/src/test/java/org/apache/gobblin/metastore/testing/TestMetastoreDatabaseServer.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class TestMetastoreDatabaseServer implements Closeable {
5252

5353
private static final String INFORMATION_SCHEMA = "information_schema";
5454
private static final String ROOT_USER = "root";
55+
private static final String ROOT_PASSWORD = "password";
5556
private static final String DROP_DATABASE_TEMPLATE = "DROP DATABASE IF EXISTS %s;";
5657
private static final String CREATE_DATABASE_TEMPLATE = "CREATE DATABASE %s CHARACTER SET = %s COLLATE = %s;";
5758
private static final String ADD_USER_TEMPLATE = "GRANT ALL ON %s.* TO '%s'@'%%';";
@@ -60,9 +61,9 @@ class TestMetastoreDatabaseServer implements Closeable {
6061
public static final String EMBEDDED_MYSQL_ENABLED_KEY = "embeddedMysqlEnabled";
6162
public static final String EMBEDDED_MYSQL_ENABLED_FULL_KEY =
6263
CONFIG_PREFIX + "." + EMBEDDED_MYSQL_ENABLED_KEY;
63-
public static final String DBUSER_NAME_KEY = "dbUserName";
64+
public static final String DBUSER_NAME_KEY = "testUser";
6465
public static final String DBUSER_NAME_FULL_KEY = CONFIG_PREFIX + "." + DBUSER_NAME_KEY;
65-
public static final String DBUSER_PASSWORD_KEY = "dbUserPassword";
66+
public static final String DBUSER_PASSWORD_KEY = "testPassword";
6667
public static final String DBUSER_PASSWORD_FULL_KEY = CONFIG_PREFIX + "." + DBUSER_PASSWORD_KEY;
6768
public static final String DBHOST_KEY = "dbHost";
6869
public static final String DBHOST_FULL_KEY = CONFIG_PREFIX + "." + DBHOST_KEY;
@@ -153,7 +154,8 @@ private MySqlJdbcUrl getBaseJdbcUrl() throws URISyntaxException {
153154
private MySqlJdbcUrl getInformationSchemaJdbcUrl() throws URISyntaxException {
154155
return getBaseJdbcUrl()
155156
.setPath(INFORMATION_SCHEMA)
156-
.setUser(ROOT_USER);
157+
.setUser(ROOT_USER)
158+
.setPassword(ROOT_PASSWORD);
157159
}
158160

159161
private Optional<Connection> getConnector(MySqlJdbcUrl jdbcUrl) throws SQLException {

‎gobblin-modules/gobblin-kafka-1/src/test/java/org/apache/gobblin/kafka/writer/Kafka1DataWriterTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646

4747

4848
@Slf4j
49+
@Test( groups = {"disabledOnCI"} )
4950
public class Kafka1DataWriterTest {
5051

5152

‎gobblin-modules/gobblin-kafka-1/src/test/java/org/apache/gobblin/metrics/reporter/KafkaKeyValueProducerPusherTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
/**
4040
* Test {@link KafkaKeyValueProducerPusher}.
4141
*/
42+
@Test( groups = {"disabledOnCI"} )
4243
public class KafkaKeyValueProducerPusherTest {
4344
public static final String TOPIC = KafkaKeyValueProducerPusherTest.class.getSimpleName();
4445

‎gobblin-modules/gobblin-kafka-1/src/test/java/org/apache/gobblin/metrics/reporter/KafkaProducerPusherTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
/**
3838
* Test {@link org.apache.gobblin.metrics.kafka.KafkaProducerPusher}.
3939
*/
40+
@Test( groups = {"disabledOnCI"} )
4041
public class KafkaProducerPusherTest {
4142
public static final String TOPIC = KafkaProducerPusherTest.class.getSimpleName();
4243

0 commit comments

Comments
 (0)