Skip to content

Commit de56df8

Browse files
shreddedbaconshreddedbacon
committed
feat: initial pass at service cleanup
1 parent a429aff commit de56df8

File tree

5 files changed

+347
-0
lines changed

5 files changed

+347
-0
lines changed

‎cmd/identify_ingress_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"time"
99

1010
"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
11+
"github.com/uselagoon/build-deploy-tool/internal/helpers"
1112
)
1213

1314
func TestIdentifyRoute(t *testing.T) {
@@ -530,6 +531,9 @@ func TestIdentifyRoute(t *testing.T) {
530531
if string(retJSON) != tt.wantJSON {
531532
t.Errorf("returned autogen %v doesn't match want %v", string(retJSON), tt.wantJSON)
532533
}
534+
t.Cleanup(func() {
535+
helpers.UnsetEnvVars(nil)
536+
})
533537
})
534538
}
535539
}

‎cmd/identify_services.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package cmd
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
7+
"github.com/spf13/cobra"
8+
generator "github.com/uselagoon/build-deploy-tool/internal/generator"
9+
"github.com/uselagoon/build-deploy-tool/internal/helpers"
10+
)
11+
12+
type identifyServices struct {
13+
Name string `json:"name"`
14+
Type string `json:"type"`
15+
}
16+
17+
var servicesIdentify = &cobra.Command{
18+
Use: "services",
19+
Aliases: []string{"s"},
20+
Short: "Identify services that this build would create",
21+
RunE: func(cmd *cobra.Command, args []string) error {
22+
generator, err := generatorInput(false)
23+
if err != nil {
24+
return err
25+
}
26+
ret, _, err := IdentifyServices(generator)
27+
if err != nil {
28+
return err
29+
}
30+
retJSON, _ := json.Marshal(ret)
31+
fmt.Println(string(retJSON))
32+
return nil
33+
},
34+
}
35+
36+
// IdentifyServices identifies services that this build would create
37+
func IdentifyServices(g generator.GeneratorInput) ([]string, []identifyServices, error) {
38+
lagoonBuild, err := generator.NewGenerator(
39+
g,
40+
)
41+
if err != nil {
42+
return nil, nil, err
43+
}
44+
45+
services := []string{}
46+
serviceTypes := []identifyServices{}
47+
for _, service := range lagoonBuild.BuildValues.Services {
48+
if service.Type != "" {
49+
services = helpers.AppendIfMissing(services, service.OverrideName)
50+
serviceTypes = AppendIfMissing(serviceTypes, identifyServices{
51+
Name: service.OverrideName,
52+
Type: service.Type,
53+
})
54+
}
55+
}
56+
return services, serviceTypes, nil
57+
}
58+
59+
func init() {
60+
identifyCmd.AddCommand(servicesIdentify)
61+
}
62+
63+
func AppendIfMissing(slice []identifyServices, i identifyServices) []identifyServices {
64+
for _, ele := range slice {
65+
if ele.Name == i.Name {
66+
return slice
67+
}
68+
}
69+
return append(slice, i)
70+
}

‎cmd/identify_services_test.go‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package cmd
2+
3+
import (
4+
"os"
5+
"reflect"
6+
"testing"
7+
"time"
8+
9+
"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
10+
"github.com/uselagoon/build-deploy-tool/internal/helpers"
11+
)
12+
13+
func TestIdentifyServices(t *testing.T) {
14+
type args struct {
15+
alertContact string
16+
statusPageID string
17+
projectName string
18+
environmentName string
19+
branch string
20+
prNumber string
21+
prHeadBranch string
22+
prBaseBranch string
23+
environmentType string
24+
buildType string
25+
activeEnvironment string
26+
standbyEnvironment string
27+
cacheNoCache string
28+
serviceID string
29+
secretPrefix string
30+
projectVars string
31+
envVars string
32+
lagoonVersion string
33+
lagoonYAML string
34+
valuesFilePath string
35+
templatePath string
36+
}
37+
tests := []struct {
38+
name string
39+
args args
40+
want []string
41+
wantServices []identifyServices
42+
wantErr bool
43+
}{
44+
{
45+
name: "test1 check LAGOON_FASTLY_SERVICE_IDS with secret no values",
46+
args: args{
47+
alertContact: "alertcontact",
48+
statusPageID: "statuspageid",
49+
projectName: "example-project",
50+
environmentName: "main",
51+
environmentType: "production",
52+
buildType: "branch",
53+
lagoonVersion: "v2.7.x",
54+
branch: "main",
55+
projectVars: `[{"name":"LAGOON_SYSTEM_ROUTER_PATTERN","value":"${service}-${project}-${environment}.example.com","scope":"internal_system"},{"name":"LAGOON_FASTLY_SERVICE_IDS","value":"example.com:service-id:true:annotationscom","scope":"build"}]`,
56+
envVars: `[]`,
57+
lagoonYAML: "../test-resources/identify-ingress/test1/lagoon.yml",
58+
templatePath: "../test-resources/output",
59+
},
60+
want: []string{"node"},
61+
wantServices: []identifyServices{{Name: "node", Type: "node"}},
62+
},
63+
{
64+
name: "test16 autogenerated routes where lagoon.name of service does not match service names",
65+
args: args{
66+
alertContact: "alertcontact",
67+
statusPageID: "statuspageid",
68+
projectName: "content-example-com",
69+
environmentName: "feature-migration",
70+
environmentType: "development",
71+
buildType: "branch",
72+
lagoonVersion: "v2.7.x",
73+
branch: "feature/migration",
74+
projectVars: `[{"name":"LAGOON_SYSTEM_ROUTER_PATTERN","value":"${environment}.${project}.example.com","scope":"internal_system"}]`,
75+
envVars: `[]`,
76+
lagoonYAML: "../test-resources/identify-ingress/test16/lagoon.yml",
77+
templatePath: "../test-resources/output",
78+
},
79+
want: []string{"cli", "nginx-php", "mariadb", "redis"},
80+
wantServices: []identifyServices{
81+
{Name: "cli", Type: "cli-persistent"},
82+
{Name: "nginx-php", Type: "nginx-php-persistent"},
83+
{Name: "mariadb", Type: "mariadb-dbaas"},
84+
{Name: "redis", Type: "redis"},
85+
},
86+
},
87+
}
88+
for _, tt := range tests {
89+
t.Run(tt.name, func(t *testing.T) {
90+
// set the environment variables from args
91+
err := os.Setenv("MONITORING_ALERTCONTACT", tt.args.alertContact)
92+
if err != nil {
93+
t.Errorf("%v", err)
94+
}
95+
err = os.Setenv("MONITORING_STATUSPAGEID", tt.args.statusPageID)
96+
if err != nil {
97+
t.Errorf("%v", err)
98+
}
99+
err = os.Setenv("PROJECT", tt.args.projectName)
100+
if err != nil {
101+
t.Errorf("%v", err)
102+
}
103+
err = os.Setenv("ENVIRONMENT", tt.args.environmentName)
104+
if err != nil {
105+
t.Errorf("%v", err)
106+
}
107+
err = os.Setenv("BRANCH", tt.args.branch)
108+
if err != nil {
109+
t.Errorf("%v", err)
110+
}
111+
err = os.Setenv("LAGOON_GIT_BRANCH", tt.args.branch)
112+
if err != nil {
113+
t.Errorf("%v", err)
114+
}
115+
err = os.Setenv("PR_NUMBER", tt.args.prNumber)
116+
if err != nil {
117+
t.Errorf("%v", err)
118+
}
119+
err = os.Setenv("PR_HEAD_BRANCH", tt.args.prHeadBranch)
120+
if err != nil {
121+
t.Errorf("%v", err)
122+
}
123+
err = os.Setenv("PR_BASE_BRANCH", tt.args.prBaseBranch)
124+
if err != nil {
125+
t.Errorf("%v", err)
126+
}
127+
err = os.Setenv("ENVIRONMENT_TYPE", tt.args.environmentType)
128+
if err != nil {
129+
t.Errorf("%v", err)
130+
}
131+
err = os.Setenv("BUILD_TYPE", tt.args.buildType)
132+
if err != nil {
133+
t.Errorf("%v", err)
134+
}
135+
err = os.Setenv("ACTIVE_ENVIRONMENT", tt.args.activeEnvironment)
136+
if err != nil {
137+
t.Errorf("%v", err)
138+
}
139+
err = os.Setenv("STANDBY_ENVIRONMENT", tt.args.standbyEnvironment)
140+
if err != nil {
141+
t.Errorf("%v", err)
142+
}
143+
err = os.Setenv("LAGOON_FASTLY_NOCACHE_SERVICE_ID", tt.args.cacheNoCache)
144+
if err != nil {
145+
t.Errorf("%v", err)
146+
}
147+
err = os.Setenv("LAGOON_PROJECT_VARIABLES", tt.args.projectVars)
148+
if err != nil {
149+
t.Errorf("%v", err)
150+
}
151+
err = os.Setenv("LAGOON_ENVIRONMENT_VARIABLES", tt.args.envVars)
152+
if err != nil {
153+
t.Errorf("%v", err)
154+
}
155+
err = os.Setenv("LAGOON_VERSION", tt.args.lagoonVersion)
156+
if err != nil {
157+
t.Errorf("%v", err)
158+
}
159+
generator, err := generatorInput(false)
160+
if err != nil {
161+
t.Errorf("%v", err)
162+
}
163+
generator.LagoonYAML = tt.args.lagoonYAML
164+
// add dbaasclient overrides for tests
165+
generator.DBaaSClient = dbaasclient.NewClient(dbaasclient.Client{
166+
RetryMax: 5,
167+
RetryWaitMin: time.Duration(10) * time.Millisecond,
168+
RetryWaitMax: time.Duration(50) * time.Millisecond,
169+
})
170+
got, got2, err := IdentifyServices(generator)
171+
if (err != nil) != tt.wantErr {
172+
t.Errorf("IdentifyServices() error = %v, wantErr %v", err, tt.wantErr)
173+
return
174+
}
175+
if !reflect.DeepEqual(got, tt.want) {
176+
t.Errorf("IdentifyServices() = %v, want %v", got, tt.want)
177+
}
178+
if !reflect.DeepEqual(got2, tt.wantServices) {
179+
t.Errorf("IdentifyServices() = %v, want %v", got2, tt.wantServices)
180+
}
181+
t.Cleanup(func() {
182+
helpers.UnsetEnvVars(nil)
183+
})
184+
})
185+
}
186+
}

‎internal/helpers/helpers.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,3 +219,12 @@ func DeepCopy(src, dist interface{}) (err error) {
219219
}
220220
return gob.NewDecoder(&buf).Decode(dist)
221221
}
222+
223+
func AppendIfMissing(slice []string, i string) []string {
224+
for _, ele := range slice {
225+
if ele == i {
226+
return slice
227+
}
228+
}
229+
return append(slice, i)
230+
}

‎legacy/build-deploy-docker-compose.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1757,6 +1757,84 @@ set +x
17571757
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
17581758
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "deploymentApplyComplete" "Applying Deployments" "false"
17591759
previousStepEnd=${currentStepEnd}
1760+
beginBuildStep "Service/Deployment Cleanup" "cleanupServices"
1761+
1762+
##############################################
1763+
### CLEANUP services which have been removed from docker-compose.yaml
1764+
##############################################s
1765+
1766+
set +x
1767+
# collect the current routes, its possible to exclude ingress by adding a label 'route.lagoon.sh/remove=false' and it won't get deleted
1768+
CURRENT_SERVICES=$(kubectl -n ${NAMESPACE} get deployments -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs)
1769+
# collect the routes that Lagoon thinks it should have based on the .lagoon.yml and any routes that have come from the api
1770+
# using the build-deploy-tool generator
1771+
SERVICES_TO_JSON=$(build-deploy-tool identify services | jq -r '.[]')
1772+
1773+
MATCHED_SERVICE=false
1774+
DELETE_SERVICE=()
1775+
# loop over the routes from kubernetes
1776+
for EXIST_SERVICE in ${CURRENT_SERVICES}; do
1777+
# loop over the routes that Lagoon thinks it should have
1778+
for SERVICE in ${SERVICES_TO_JSON}; do
1779+
if [ "${EXIST_SERVICE}" == "${SERVICE}" ]; then
1780+
MATCHED_SERVICE=true
1781+
continue
1782+
fi
1783+
done
1784+
if [ "${MATCHED_SERVICE}" != "true" ]; then
1785+
DELETE_SERVICE+=($EXIST_SERVICE)
1786+
fi
1787+
MATCHED_SERVICE=false
1788+
done
1789+
1790+
SERVICE_CLEANUP_WARNINGS="false"
1791+
if [ ${#DELETE_SERVICE[@]} -ne 0 ]; then
1792+
SERVICE_CLEANUP_WARNINGS="true"
1793+
((++BUILD_WARNING_COUNT))
1794+
echo ">> Lagoon detected services that have been removed from the docker-compose file"
1795+
if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" != enabled ]; then
1796+
echo "> You can remove these in the next build by setting the flag 'LAGOON_FEATURE_FLAG_CLEANUP_REMOVED_LAGOON_SERVICES=enabled' as a GLOBAL scoped variable to this environment or project"
1797+
fi
1798+
for DS in ${DELETE_SERVICE[@]}
1799+
do
1800+
if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then
1801+
echo ">> Removing deployment ${DS}"
1802+
if kubectl -n ${NAMESPACE} get deployments ${DS} &> /dev/null; then
1803+
kubectl -n ${NAMESPACE} delete deployment ${DS}
1804+
fi
1805+
if kubectl -n ${NAMESPACE} get service ${DS} &> /dev/null; then
1806+
echo ">>> Removing associated service ${DS}"
1807+
kubectl -n ${NAMESPACE} delete service ${DS}
1808+
fi
1809+
if kubectl -n ${NAMESPACE} get ingress ${DS} &> /dev/null; then
1810+
echo ">>> Removing associated ingress ${DS}"
1811+
kubectl -n ${NAMESPACE} delete ingress ${DS}
1812+
fi
1813+
if kubectl -n ${NAMESPACE} get pvc ${DS} &> /dev/null; then
1814+
echo ">>> Removing associated persistent volume ${DS}"
1815+
kubectl -n ${NAMESPACE} delete pvc ${DS}
1816+
fi
1817+
#delete anything else?
1818+
else
1819+
echo ">> The deployment '${DS}' would be removed"
1820+
if kubectl -n ${NAMESPACE} get service ${DS} &> /dev/null; then
1821+
echo ">>> The associated service '${DS}' would be removed"
1822+
fi
1823+
if kubectl -n ${NAMESPACE} get ingress ${DS} &> /dev/null; then
1824+
echo ">>> The associated ingress '${DS}' would be removed"
1825+
fi
1826+
if kubectl -n ${NAMESPACE} get pvc ${DS} &> /dev/null; then
1827+
echo ">>> The associated persistent volume '${DS}' would be removed"
1828+
fi
1829+
fi
1830+
done
1831+
else
1832+
echo "No service cleanup required"
1833+
fi
1834+
1835+
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
1836+
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "serviceCleanupComplete" "Service/Deployment Cleanup" "${SERVICE_CLEANUP_WARNINGS}"
1837+
previousStepEnd=${currentStepEnd}
17601838
beginBuildStep "Cronjob Cleanup" "cleaningUpCronjobs"
17611839
set -x
17621840

0 commit comments

Comments
 (0)