Skip to content

Commit 0585c5e

Browse files
danielbwatsondanielbwatson
committed
Adding functional tests to Metacat.
 - Before running the tests run 'buildWarImage' to build a tomcat image of the Metacat war. - Run 'functionalTests' to launch a test cluster, using docker-compose, and run the functional test suite against the server.
1 parent 41f0268 commit 0585c5e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+163679
-26
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,6 @@ atlassian-ide-plugin.xml
6767

6868
# Data created by elastic search tests
6969
metacat-main/data/
70+
71+
# Functional tests
72+
metacat-functional-tests/metacat-test-cluster/build/

‎.travis.yml

+16-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,21 @@
11
language: java
22
jdk:
3-
- oraclejdk8
4-
sudo: false
5-
install: ./installViaTravis.sh
3+
- oraclejdk8
4+
sudo: required
5+
dist: trusty
6+
services:
7+
- docker
8+
install:
9+
# Update docker-engine using Ubuntu 'trusty' apt repo
10+
- curl -sSL "https://get.docker.com/gpg" | sudo -E apt-key add -
11+
- echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | sudo tee -a /etc/apt/sources.list
12+
- sudo apt-get update
13+
- sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --assume-yes install docker-engine
14+
- docker version
15+
# Update docker-compose via pip
16+
- sudo pip install docker-compose
17+
- docker-compose version
18+
- ./installViaTravis.sh
619
script: ./buildViaTravis.sh
720
env:
821
global:

‎buildViaTravis.sh

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,21 @@
33

44
if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
55
echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]"
6-
./gradlew build
6+
./gradlew build buildWarImage functionalTest
77
elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then
88
echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']'
9-
./gradlew -Prelease.travisci=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" build snapshot
9+
./gradlew -Prelease.travisci=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" build buildWarImage functionalTest snapshot
1010
elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then
1111
echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']'
1212
case "$TRAVIS_TAG" in
1313
*-rc\.*)
14-
./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" candidate
14+
./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" buildWarImage functionalTest candidate
1515
;;
1616
*)
17-
./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" final
17+
./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" buildWarImage functionalTest final
1818
;;
1919
esac
2020
else
2121
echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']'
22-
./gradlew build
22+
./gradlew build buildWarImage functionalTest
2323
fi

‎metacat-functional-tests/build.gradle

+281
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
/*
2+
* Copyright 2016 Netflix, Inc.
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an "AS IS" BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License.
12+
*/
13+
14+
import com.bmuschko.gradle.docker.tasks.DockerInfo
15+
import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
16+
import com.bmuschko.gradle.docker.tasks.image.Dockerfile
17+
18+
ext {
19+
if (project.hasProperty('startCluster')) {
20+
startCluster = Boolean.getBoolean('startCluster')
21+
} else {
22+
startCluster = true
23+
}
24+
logger.info("metacat-functional-tests: start cluster = '${startCluster}'")
25+
26+
if (project.hasProperty('stopCluster')) {
27+
stopCluster = Boolean.getBoolean('stopCluster')
28+
} else {
29+
stopCluster = true
30+
}
31+
logger.info("metacat-functional-tests: stop cluster = '${stopCluster}'")
32+
33+
if (project.hasProperty('dockerHost')) {
34+
dockerHost = project['dockerHost']
35+
} else if (System.getenv('DOCKER_HOST')) {
36+
dockerHost = System.getenv('DOCKER_HOST')?.replace('tcp:', 'https:')
37+
} else {
38+
dockerHost = 'http://localhost:2375'
39+
}
40+
logger.info("metacat-functional-tests: docker host '${dockerHost}'")
41+
42+
if (project.hasProperty('dockerIp')) {
43+
dockerIp = project['dockerIp']
44+
} else if (System.getenv('DOCKER_IP')) {
45+
dockerIp = System.getenv('DOCKER_IP')
46+
} else if (System.getenv('DOCKER_HOST')?.startsWith('tcp') || System.getenv('DOCKER_HOST')?.startsWith('http')) {
47+
dockerIp = (System.getenv('DOCKER_HOST') =~ /:\/\/([0-9\.]+):/)[0][1]
48+
} else {
49+
dockerIp = 'localhost'
50+
}
51+
logger.info("metacat-functional-tests: docker ip '${dockerIp}'")
52+
53+
if (project.hasProperty('dockerCertPath')) {
54+
dockerCertPath = new File(project['dockerCertPath'] as String)
55+
} else if (System.getenv('DOCKER_CERT_PATH')) {
56+
dockerCertPath = new File(System.getenv('DOCKER_CERT_PATH'))
57+
} else {
58+
dockerCertPath = null
59+
}
60+
logger.info("metacat-functional-tests: docker cert path '${dockerCertPath}'")
61+
62+
clusterWorkingDir = file("${rootDir}/metacat-functional-tests/metacat-test-cluster")
63+
}
64+
65+
configurations {
66+
warproject
67+
functionalTestCompile.extendsFrom testCompile
68+
functionalTestRuntime.extendsFrom testRuntime
69+
70+
all*.exclude group: 'javax.ws.rs', module: 'javax.ws.rs-api'
71+
}
72+
73+
sourceSets {
74+
functionalTest {
75+
compileClasspath += main.output + test.output
76+
runtimeClasspath += main.output + test.output
77+
78+
java.srcDir file('src/functionalTest/java')
79+
groovy.srcDir file('src/functionalTest/groovy')
80+
resources.srcDir file('src/functionalTest/resources')
81+
}
82+
}
83+
84+
buildscript {
85+
repositories {
86+
jcenter()
87+
}
88+
89+
dependencies {
90+
classpath 'com.bmuschko:gradle-docker-plugin:3.0.1'
91+
}
92+
}
93+
94+
apply plugin: 'com.bmuschko.docker-remote-api'
95+
96+
dependencies {
97+
warproject project(path: ':metacat-server', configuration: 'archives')
98+
99+
// Test that metacat-client can be used with jersey 1
100+
testCompile(project(':metacat-client')) {
101+
exclude group: 'javax.ws.rs', module: 'javax.ws.rs-api'
102+
exclude group: 'org.glassfish.jersey.core'
103+
}
104+
testCompile 'com.sun.jersey:jersey-client:1.19.1'
105+
106+
testCompile project(':metacat-common-server')
107+
testCompile 'javax.ws.rs:jsr311-api:1.1.1'
108+
testCompile "com.facebook.presto:presto-hive-hadoop2:${presto_version}"
109+
}
110+
111+
docker {
112+
url = project.ext.dockerHost
113+
certPath = project.ext.dockerCertPath
114+
}
115+
116+
task expandWar(type: Copy) {
117+
dependsOn ':metacat-server:war'
118+
119+
from { configurations.warproject.collect { zipTree(it) } }
120+
into file("${buildDir}/metacat-war-expanded/ROOT")
121+
}
122+
123+
task dockerInfo(type: DockerInfo)
124+
125+
task createWarDockerfile(type: Dockerfile) {
126+
dependsOn 'expandWar'
127+
128+
destFile = project.file('build/metacat-war-expanded/Dockerfile')
129+
from 'tomcat:8.0-jre8'
130+
maintainer 'Netflix OSS "[email protected]"'
131+
volume '/usr/local/tomcat/logs', '/etc/metacat'
132+
runCommand 'rm -rf /usr/local/tomcat/webapps'
133+
copyFile './ROOT', '/usr/local/tomcat/webapps/ROOT'
134+
}
135+
136+
task buildWarImage(type: DockerBuildImage) {
137+
dependsOn 'createWarDockerfile'
138+
inputDir = createWarDockerfile.destFile.parentFile
139+
tag = 'netflix_metacat_test/metacat_server'
140+
}
141+
142+
task startMetacatCluster(type: Exec) {
143+
workingDir = clusterWorkingDir
144+
145+
environment 'PATH', '/usr/local/bin:/usr/bin:/bin'
146+
environment 'DOCKER_HOST', project.ext.dockerHost?.replace('http:', 'tcp:').replace('https', 'tcp:')
147+
if (project.ext.dockerCertPath) {
148+
environment 'DOCKER_CERT_PATH', project.ext.dockerCertPath
149+
environment 'DOCKER_TLS_VERIFY', '1'
150+
}
151+
152+
commandLine './startCluster.sh'
153+
}
154+
155+
task stopMetacatCluster(type: Exec) {
156+
workingDir = clusterWorkingDir
157+
158+
environment 'PATH', '/usr/local/bin:/usr/bin:/bin'
159+
environment 'DOCKER_HOST', project.ext.dockerHost?.replace('http:', 'tcp:').replace('https', 'tcp:')
160+
if (project.ext.dockerCertPath) {
161+
environment 'DOCKER_CERT_PATH', project.ext.dockerCertPath
162+
environment 'DOCKER_TLS_VERIFY', '1'
163+
}
164+
165+
commandLine './stopCluster.sh'
166+
}
167+
168+
task metacatPorts {
169+
if (project.ext.startCluster) {
170+
dependsOn 'startMetacatCluster'
171+
}
172+
ext.http_port = null
173+
ext.metacat_hive_thrift_port = null
174+
ext.metacat_s3_thrift_port = null
175+
ext.hive_thrift_port = null
176+
177+
doLast {
178+
new ByteArrayOutputStream().withStream { os ->
179+
exec {
180+
workingDir = clusterWorkingDir
181+
182+
environment 'DOCKER_HOST', project.ext.dockerHost?.replace('http:', 'tcp:').replace('https', 'tcp:')
183+
if (project.ext.dockerCertPath) {
184+
environment 'DOCKER_CERT_PATH', project.ext.dockerCertPath
185+
environment 'DOCKER_TLS_VERIFY', '1'
186+
}
187+
188+
commandLine './printDockerPort.sh', project.ext.dockerIp, 'label=com.netflix.metacat.oss.test.war', 8080
189+
standardOutput = os
190+
}
191+
ext.http_port = os.toString().trim()
192+
logger.info("metacat-functional-tests: metacat http_port '{}'", ext.http_port)
193+
}
194+
195+
new ByteArrayOutputStream().withStream { os ->
196+
exec {
197+
workingDir = clusterWorkingDir
198+
199+
environment 'DOCKER_HOST', project.ext.dockerHost?.replace('http:', 'tcp:').replace('https', 'tcp:')
200+
if (project.ext.dockerCertPath) {
201+
environment 'DOCKER_CERT_PATH', project.ext.dockerCertPath
202+
environment 'DOCKER_TLS_VERIFY', '1'
203+
}
204+
205+
commandLine './printDockerPort.sh', project.ext.dockerIp, 'label=com.netflix.metacat.oss.test.war', 12001
206+
standardOutput = os
207+
}
208+
ext.metacat_hive_thrift_port = os.toString().trim()
209+
logger.info("metacat-functional-tests: metacat thrift_port '{}'", ext.metacat_hive_thrift_port)
210+
}
211+
212+
new ByteArrayOutputStream().withStream { os ->
213+
exec {
214+
workingDir = clusterWorkingDir
215+
216+
environment 'DOCKER_HOST', project.ext.dockerHost?.replace('http:', 'tcp:').replace('https', 'tcp:')
217+
if (project.ext.dockerCertPath) {
218+
environment 'DOCKER_CERT_PATH', project.ext.dockerCertPath
219+
environment 'DOCKER_TLS_VERIFY', '1'
220+
}
221+
222+
commandLine './printDockerPort.sh', project.ext.dockerIp, 'label=com.netflix.metacat.oss.test.war', 12002
223+
standardOutput = os
224+
}
225+
ext.metacat_s3_thrift_port = os.toString().trim()
226+
logger.info("metacat-functional-tests: metacat thrift_port '{}'", ext.metacat_s3_thrift_port)
227+
}
228+
229+
new ByteArrayOutputStream().withStream { os ->
230+
exec {
231+
workingDir = clusterWorkingDir
232+
233+
environment 'DOCKER_HOST', project.ext.dockerHost?.replace('http:', 'tcp:').replace('https', 'tcp:')
234+
if (project.ext.dockerCertPath) {
235+
environment 'DOCKER_CERT_PATH', project.ext.dockerCertPath
236+
environment 'DOCKER_TLS_VERIFY', '1'
237+
}
238+
239+
commandLine './printDockerPort.sh', project.ext.dockerIp, 'label=com.netflix.metacat.oss.test.hive', 9083
240+
standardOutput = os
241+
}
242+
ext.hive_thrift_port = os.toString().trim()
243+
logger.info("metacat-functional-tests: metacat hive_thrift_port '{}'", ext.hive_thrift_port)
244+
}
245+
}
246+
}
247+
248+
task functionalTest(type: Test) {
249+
testClassesDir = sourceSets.functionalTest.output.classesDir
250+
classpath = sourceSets.functionalTest.runtimeClasspath
251+
outputs.upToDateWhen { false }
252+
253+
doFirst {
254+
def properties = [
255+
'metacat_docker_ip' : project.ext.dockerIp as String,
256+
'metacat_http_port' : metacatPorts.http_port as String,
257+
'metacat_hive_thrift_port' : metacatPorts.metacat_hive_thrift_port as String,
258+
'metacat_s3_thrift_port' : metacatPorts.metacat_s3_thrift_port as String,
259+
'hive_thrift_port' : metacatPorts.hive_thrift_port as String,
260+
'org.slf4j.simpleLogger.defaultLogLevel': 'debug'
261+
]
262+
System.properties.stringPropertyNames().findAll { it?.toLowerCase()?.contains("metacat") }.each {
263+
properties[it] = System.getProperty(it)
264+
}
265+
systemProperties = properties
266+
}
267+
268+
testLogging {
269+
showStandardStreams = true
270+
}
271+
272+
if (project.ext.startCluster) {
273+
dependsOn 'startMetacatCluster', 'metacatPorts'
274+
} else {
275+
dependsOn 'metacatPorts'
276+
}
277+
278+
if (project.ext.stopCluster) {
279+
finalizedBy 'stopMetacatCluster'
280+
}
281+
}

0 commit comments

Comments
 (0)