Skip to content

Commit ff6238d

Browse files
authored
Remove redundant script from Travis build and improve log information (#1140)
* Improve Travis build log information * Improve Travis build log information * Improve Travis build log information * Improve Travis builds * Improve Travis builds * improve task information for logs * improve task information for logs * improve task information for logs * Add spew for export commands that get SIGINT faults * Add spew for export commands that get SIGINT faults * Add trace around intermittent failings on export project * Cleanup changes and add comments and TODOs
1 parent d653592 commit ff6238d

File tree

13 files changed

+221
-96
lines changed

13 files changed

+221
-96
lines changed

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ _testmain.go
2424
*.prof
2525

2626

27-
#OS X
27+
# OS X
2828
**/.DS_Store
2929

3030
#binaries
@@ -43,6 +43,7 @@ Packages/
4343
.gradle
4444
.gogradle
4545
build/
46+
release/
4647

4748
#emacs
4849
*~

.travis.yml

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,33 +36,64 @@ notifications:
3636
# travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results
3737
secure: "NaDCtHwbFUEy0h0u/QrAt395o0/9FstJ5/Tv4uBSLCBdaUNU+0q6/RXVH76dKbujoxrLded2gyeJSAbI4kl4SM7FyilSHzKTikF0VsgM3G5veh65tVY5ztsvl40hDmK2E/PmWI3HkZNeWJO32WjraqPmhSWL0BZOWhA/4gJSCIsLnNRQXmpZoDqC7V8IG1BWYGwd5qcNkEjItLjYCUy1s2K8bj3QmhshYRVLTNbXxh0yJVKjM04I+bWhMXqvL8GS55qeicJ3fUwm5g1PDVCtzxLwAHCPo2jusjtNSnV+BaZgMBSahv3MD7ApxfF58e1buejggH3qZAhNGccC4bYJCahqVv/KKoA10kO6exH5iGwlHPWQTjMuF++PHmAk+FeQ1jh+JoUSBnIHExPnaD4CQIArHvUuQom+WJBnOz+L1H755VyPGzeDQ+ZUDlhOiQ6CDP/sqaRH3Wmo4IzhsqTsLaQs0dS1YeEWOS8gcawwNTVf/WtqiedYu24rQN9RiUXelRoEaXJy/pldb2KpyQRo2hbKoStDDB5fEij1xyGsj4kSXZ8uk5G10oDGnZYd+okduzJazN8wr0flrLvH32DRgDZDcG/D90I19JmqqfqbcG/SEmhxTikHZeVQd5DfWcqssLvWNrtQa/EJgAQkmZ2gaSsZ73AvPk/KyMV30+m8xTU="
3838

39+
# The complete sequence of phases of the lifecycle:
40+
# ---------------------------------------------------------
41+
# before_install
42+
# install
43+
# before_script
44+
# script
45+
# before_cache (OPTIONAL) (if and only if caching is effective)
46+
# after_success (or after_failure)
47+
# before_deploy (OPTIONAL) (if and only if deployment is active)
48+
# deploy (OPTIONAL)
49+
# after_deploy (OPTIONAL) (if and only if deployment is active)
50+
# after_script
51+
3952
before_install:
4053
- "./tools/travis/cloneutils.sh"
54+
- sudo apt-get install tree
55+
56+
# Install tools we use for linting and unit/integration testing
57+
# TODO: build.gradle (gradlew) already installs 'golint' and runs linting
58+
# in the `goLint` task; see if we can remove here.
59+
# TODO: The "gogradle" plugin in build.gradle (gradlew) already provides a
60+
# task called "goTest" which runs all `unit` tests. See if we can use that
61+
# instead and only run `integration` tests here.
4162
install:
4263
- export DEPLOY_BUILD_READY=false
4364
- go get -u golang.org/x/lint/golint
4465
- go get -u github.com/stretchr/testify
66+
67+
# Identify Golang files that are not properly formatted
68+
# TODO: See if we can create custom tasks to perform this in build.gradle
69+
# as a custom `goFmt` task
4570
before_script:
4671
- GO_FILES=$(find . -iname '*.go' -type f -not -path "./wski18n/i18n_resources.go")
4772
- export BAD_GO=$(gofmt -s -l $(echo $GO_FILES))
4873
- echo "["$BAD_GO"]"
4974
#- test -z "$BAD_GO"
5075
#- test -z "$(gofmt -s -l $(echo $GO_FILES))"
5176

77+
# Clone and build the OpenWhisk platform (i.e., apache/openwhisk)
78+
# and then use Ansible to deploy it and prepare it to run our
79+
# integration tests.
80+
# TODO: Do not use Makefile for running integration tests (in script.sh);
81+
# change to using gradle wrapper
5282
script:
53-
- echo $TRAVIS
54-
- echo $TRAVIS_PULL_REQUEST
55-
- echo $TRAVIS_SECURE_ENV_VARS
56-
- printenv
83+
- echo "Installing and building OpenWhisk platform using Ansible..."
84+
- "./tools/travis/showenv.sh"
5785
- "./tools/travis/script.sh"
86+
5887
after_success:
5988
- DEPLOY_BUILD_READY=true
6089
- if [ "$TRAVIS_EVENT_TYPE" == "cron" ] ; then
6190
export DEPLOY_BUILD_READY=false;
6291
fi
6392

64-
after_script:
65-
- make clean
93+
# If this is a "tagged" GitHub build, then use Gradle Wrapper
94+
# to build binaries for all supported OS platforms and architectures.
95+
# The result would be a "/build" directory that has a subdirectory named
96+
# for each supported GOOS/GOARCH
6697
before_deploy:
6798
- export build_file_name=wskdeploy
6899
- go get github.com/inconshreveable/mousetrap
@@ -72,10 +103,14 @@ before_deploy:
72103
export GIT_TAG=$TRAVIS_TAG;
73104
export TAG=true;
74105
fi
106+
- echo "Calling Gradle Wrapper task to build and 'releaseBinaries'..."
107+
- "./tools/travis/showenv.sh"
75108
- ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
76-
- "./tools/travis/build_tag_releases.sh $build_file_name $GIT_TAG"
109+
- echo "Build binaries and release archives created:"
110+
- tree /build
111+
- tree /release
77112
- export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)"
78-
- echo "Deploying $RELEASE_PKG_FILE to GitHub releases."
113+
- echo "Deploying "$RELEASE_PKG_FILE" to GitHub releases..."
79114
- if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_EVENT_TYPE" == "push" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then
80115
git config --global user.email "[email protected]";
81116
git config --global user.name "Travis CI";
@@ -84,11 +119,11 @@ before_deploy:
84119
GIT_COMMITTER_DATE="$(git show --format=%aD | head -1)" git tag $GIT_TAG -a -m "Generated tag from Travis CI build $TRAVIS_BUILD_NUMBER";
85120
git push -f -q https://[email protected]/apache/openwhisk-wskdeploy $GIT_TAG;
86121
fi
87-
- echo "TRAVIS_EVENT_TYPE is "
88-
- echo $TRAVIS_EVENT_TYPE
89-
- echo "GIT_TAG is "
90-
- echo $GIT_TAG
122+
- echo "Completing before_deploy stage"
91123

124+
# Utilize two providers to create a github release and publish a docker image.
125+
# For information on the "releases" provider,
126+
# see https://docs.travis-ci.com/user/deployment/releases/
92127
deploy:
93128
- provider: releases
94129
api_key:
@@ -112,6 +147,18 @@ deploy:
112147
repo: apache/openwhisk-wskdeploy
113148
tags: true
114149
condition: "$DEPLOY_BUILD_READY = true"
150+
151+
# Clean up wskdeploy binary (single)
152+
# Fundamentally, remove both /build and /release directory trees and contents
153+
# TODO: See if this is really needed in Travis environments as the
154+
# Makefile is not used elsewhere here (especially if we do not use th
155+
# build_tag_release.sh script) as it is for Windows developers primarily
156+
# and we are building using Linux.
157+
after_script:
158+
- make clean
159+
- tree /build
160+
- tree /release
161+
115162
env:
116163
global:
117164
- BLUEMIX_APIHOST=openwhisk.ng.bluemix.net

build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ rat {
7373
storage is in "GO" format:
7474
7575
OS: linux, darwin, windows
76-
Arch: 386, amd64, s390x, ppc64le, arm
76+
Arch: 386, amd64, s390x, ppc64le, arm, arm64
7777
7878
TODO - It may be appropriate to refactor into a general purpose Platform
7979
class for all builds, then to extend with specifics needed for

cmd/export.go

Lines changed: 88 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ import (
2929
"github.com/apache/openwhisk-wskdeploy/parsers"
3030
"github.com/apache/openwhisk-wskdeploy/runtimes"
3131
"github.com/apache/openwhisk-wskdeploy/utils"
32+
"github.com/apache/openwhisk-wskdeploy/wskderrors"
3233
"github.com/apache/openwhisk-wskdeploy/wski18n"
3334
"github.com/apache/openwhisk-wskdeploy/wskprint"
35+
"github.com/davecgh/go-spew/spew"
3436
"github.com/spf13/cobra"
3537
)
3638

@@ -117,9 +119,6 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML,
117119

118120
func exportProject(projectName string, targetManifest string) error {
119121

120-
whisk.SetVerbose(utils.Flags.Verbose)
121-
whisk.SetDebug(utils.Flags.Trace)
122-
123122
maniyaml := &parsers.YAML{}
124123
maniyaml.Project.Name = projectName
125124

@@ -129,13 +128,19 @@ func exportProject(projectName string, targetManifest string) error {
129128
return err
130129
}
131130

131+
// Emit additional trace data (primarily in Travis)
132+
if utils.Flags.Trace {
133+
spew.Dump(packages)
134+
}
135+
132136
var bindings = make(map[string]whisk.Binding)
133137

134138
// iterate over each package to find managed annotations
135139
// check if "managed" annotation is attached to a package
136140
// add to export when managed project name matches with the
137141
// specified project name
138142
for _, pkg := range packages {
143+
139144
if a := pkg.Annotations.GetValue(utils.MANAGED); a != nil {
140145
// decode the JSON blob and retrieve __OW_PROJECT_NAME
141146
pa := a.(map[string]interface{})
@@ -199,59 +204,66 @@ func exportProject(projectName string, targetManifest string) error {
199204

200205
// iterate over the list of triggers to determine whether any of them part of specified managed project
201206
for _, trg := range triggers {
202-
// trigger has attached managed annotation
203-
if a := trg.Annotations.GetValue(utils.MANAGED); a != nil {
204-
// decode the JSON blob and retrieve __OW_PROJECT_NAME
205-
ta := a.(map[string]interface{})
206-
if ta[utils.OW_PROJECT_NAME] == projectName {
207-
208-
for pkgName := range maniyaml.Packages {
209-
if maniyaml.Packages[pkgName].Namespace == trg.Namespace {
210-
if maniyaml.Packages[pkgName].Triggers == nil {
211-
pkg := maniyaml.Packages[pkgName]
212-
pkg.Triggers = make(map[string]parsers.Trigger)
213-
maniyaml.Packages[pkgName] = pkg
214-
}
215207

216-
// export trigger to manifest
217-
218-
if feedname, isFeed := utils.IsFeedAction(&trg); isFeed {
219-
// check if feed name starts with namespace and workaround it
220-
// the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action
221-
// we need to transform the path from e.g.
222-
// /api/v1/namespaces/[email protected]_uspace/actions//whisk.system/alarms/interval?blocking=true
223-
// in to
224-
// /api/v1/namespaces/[email protected]_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true
225-
if strings.HasPrefix(feedname, "/") {
226-
// /whisk.system/alarms/interval -> ../../whisk.system/actions/alarms/interval
227-
prts := strings.SplitN(feedname, "/", 3)
228-
feedname = "../../" + prts[1] + "/actions/" + prts[2]
208+
// trigger has attached managed annotation
209+
if trg.Annotations != nil {
210+
if a := trg.Annotations.GetValue(utils.MANAGED); a != nil {
211+
// decode the JSON blob and retrieve __OW_PROJECT_NAME
212+
ta := a.(map[string]interface{})
213+
if ta[utils.OW_PROJECT_NAME] == projectName {
214+
215+
for pkgName := range maniyaml.Packages {
216+
if maniyaml.Packages[pkgName].Namespace == trg.Namespace {
217+
if maniyaml.Packages[pkgName].Triggers == nil {
218+
pkg := maniyaml.Packages[pkgName]
219+
pkg.Triggers = make(map[string]parsers.Trigger)
220+
maniyaml.Packages[pkgName] = pkg
229221
}
230222

231-
// export feed input parameters
232-
params := make(map[string]interface{})
233-
params["authKey"] = client.Config.AuthToken
234-
params["lifecycleEvent"] = "READ"
235-
params["triggerName"] = "/" + client.Namespace + "/" + trg.Name
236-
res, _, err := client.Actions.Invoke(feedname, params, true, true)
237-
if err != nil {
238-
return err
239-
}
240-
feedConfig := res["config"]
223+
// export trigger to manifest
224+
225+
if feedname, isFeed := utils.IsFeedAction(&trg); isFeed {
226+
// check if feed name starts with namespace and workaround it
227+
// the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action
228+
// we need to transform the path from e.g.
229+
// /api/v1/namespaces/[email protected]_uspace/actions//whisk.system/alarms/interval?blocking=true
230+
// in to
231+
// /api/v1/namespaces/[email protected]_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true
232+
if strings.HasPrefix(feedname, "/") {
233+
// /whisk.system/alarms/interval -> ../../whisk.system/actions/alarms/interval
234+
prts := strings.SplitN(feedname, "/", 3)
235+
feedname = "../../" + prts[1] + "/actions/" + prts[2]
236+
}
237+
238+
// export feed input parameters
239+
params := make(map[string]interface{})
240+
params["authKey"] = client.Config.AuthToken
241+
params["lifecycleEvent"] = "READ"
242+
params["triggerName"] = "/" + client.Namespace + "/" + trg.Name
243+
res, _, err := client.Actions.Invoke(feedname, params, true, true)
244+
if err != nil {
245+
return err
246+
}
247+
feedConfig := res["config"]
241248

242-
if feedConfig != nil {
243-
for key, val := range feedConfig.(map[string]interface{}) {
244-
if key != "startDate" {
245-
trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val})
249+
if feedConfig != nil {
250+
for key, val := range feedConfig.(map[string]interface{}) {
251+
if key != "startDate" {
252+
trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val})
253+
}
246254
}
247255
}
248256
}
249-
}
250257

251-
maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg)
258+
maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg)
259+
}
252260
}
253261
}
254262
}
263+
} else {
264+
// Emit additional trace data on common Travis failures
265+
spew.Dump(trg)
266+
return wskderrors.NewCommandError("Export", "Trigger missing annotations.")
255267
}
256268
}
257269

@@ -263,29 +275,36 @@ func exportProject(projectName string, targetManifest string) error {
263275

264276
// iterate over the list of rules to determine whether any of them is part of the manage dproject
265277
for _, rule := range rules {
278+
266279
// get rule from OW
267280
wskRule, _, _ := client.Rules.Get(rule.Name)
268-
// rule has attached managed annotation
269-
if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
270-
// decode the JSON blob and retrieve __OW_PROJECT_NAME
271-
ta := a.(map[string]interface{})
272-
if ta[utils.OW_PROJECT_NAME] == projectName {
273-
274-
for pkgName := range maniyaml.Packages {
275-
if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
276-
if maniyaml.Packages[pkgName].Rules == nil {
277-
pkg := maniyaml.Packages[pkgName]
278-
pkg.Rules = make(map[string]parsers.Rule)
279-
maniyaml.Packages[pkgName] = pkg
280-
}
281281

282-
// export rule to manifest
283-
maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
282+
if wskRule.Annotations != nil {
283+
// rule has attached managed annotation
284+
if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
285+
// decode the JSON blob and retrieve __OW_PROJECT_NAME
286+
ta := a.(map[string]interface{})
287+
if ta[utils.OW_PROJECT_NAME] == projectName {
288+
289+
for pkgName := range maniyaml.Packages {
290+
if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
291+
if maniyaml.Packages[pkgName].Rules == nil {
292+
pkg := maniyaml.Packages[pkgName]
293+
pkg.Rules = make(map[string]parsers.Rule)
294+
maniyaml.Packages[pkgName] = pkg
295+
}
296+
297+
// export rule to manifest
298+
maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
299+
}
284300
}
285301
}
286302
}
303+
} else {
304+
// Emit additional trace data on common Travis failures
305+
spew.Dump(wskRule)
306+
return wskderrors.NewCommandError("Export", "Rule missing annotations.")
287307
}
288-
289308
}
290309

291310
// API Gateway is an optional component. Export APIs only when ApigwAccessToken is configured
@@ -386,7 +405,14 @@ func exportProject(projectName string, targetManifest string) error {
386405

387406
// find exported manifest parent directory
388407
manifestDir := filepath.Dir(utils.Flags.ManifestPath)
389-
os.MkdirAll(manifestDir, os.ModePerm)
408+
errMkDir := os.MkdirAll(manifestDir, os.ModePerm)
409+
410+
// Exit if unable to create export dir. structure
411+
// TODO: This sometimes fails in Travis, perhaps retry?
412+
if errMkDir != nil {
413+
wskprint.PrintOpenWhiskError(errMkDir.Error())
414+
return errMkDir
415+
}
390416

391417
// export manifest to file
392418
parsers.Write(maniyaml, targetManifest)

go.mod

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,15 @@ go 1.15
55
require (
66
github.com/apache/openwhisk-client-go v0.0.0-20210311185314-87edc2364717
77
github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
8+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
89
github.com/fatih/color v1.10.0
910
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
10-
github.com/google/go-querystring v1.0.0 // indirect
1111
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519
1212
github.com/magiconair/properties v1.8.4 // indirect
1313
github.com/mattn/go-colorable v0.1.8
1414
github.com/mitchellh/mapstructure v1.4.1 // indirect
1515
github.com/nicksnyder/go-i18n v1.10.1
1616
github.com/nxadm/tail v1.4.8 // indirect
17-
github.com/onsi/ginkgo v1.15.0 // indirect
1817
github.com/pelletier/go-toml v1.8.1 // indirect
1918
github.com/spf13/afero v1.5.1 // indirect
2019
github.com/spf13/cast v1.3.1 // indirect
@@ -26,5 +25,4 @@ require (
2625
golang.org/x/text v0.3.5 // indirect
2726
gopkg.in/ini.v1 v1.62.0 // indirect
2827
gopkg.in/yaml.v2 v2.4.0
29-
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
3028
)

0 commit comments

Comments
 (0)