autoloader/cloudbuild.yaml at main · m-lab/autoloader · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
timeout: 1800s

options:
  env:
  - PROJECT_ID=$PROJECT_ID

steps:
- name: gcr.io/$PROJECT_ID/golang-cbif:1.20
  id: "Run all unit tests"
  args:
  - go version
  - go get -v -t ./...
  - go test -v -race ./...
  - go test -v ./...

- name: gcr.io/cloud-builders/docker
  id: "Build the autoloader docker image"
  args: [
    "build",
      "-t", "gcr.io/$PROJECT_ID/autoloader:${COMMIT_SHA}",
      "-f", "Dockerfile", "."
  ]

- name: gcr.io/cloud-builders/docker
  id: "Push the docker container to gcr.io"
  args: [
    "push", "gcr.io/$PROJECT_ID/autoloader:${COMMIT_SHA}",
  ]

- name: gcr.io/$PROJECT_ID/gcloud-jsonnet-cbif:1.1
  id: "v1/Setup kubectl credentials for autoloader"
  args:
  - gcloud container clusters get-credentials
    --project="${PROJECT_ID}" --region="$_CLUSTER_REGION" "$_CLUSTER_NAME"
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti

- name: gcr.io/$PROJECT_ID/gcloud-jsonnet-cbif:1.1
  id: "v1/Deploy the autoloader"
  args:
  - |-
    sed -i -e 's/{{IMAGE_TAG}}/'${COMMIT_SHA}'/'
           -e 's/{{PROJECT_ID}}/'${PROJECT_ID}'/'
           -e 's/{{BQ_PROJECT}}/'${PROJECT_ID}'/'
           -e 's/{{VIEW_PROJECT}}/'$_VIEW_PROJECT'/'
           -e 's/{{GCS_PROJECT}}/'$_GCS_PROJECT'/'
           -e 's/{{MLAB_BUCKET}}/'$_MLAB_BUCKET'/'
           -e 's/{{BUCKETS}}/'$_BUCKETS'/'
           k8s/autoloader.yaml
  - kubectl apply -f k8s/autoloader.yaml
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti

- name: gcr.io/$PROJECT_ID/gcloud-jsonnet-cbif:1.1
  id: "v1/Deploy cron jobs"
  args:
  # The "hourly" job runs every 3 hours and loads the data for the last day.
  - cp k8s/autoloader-cronjob.yaml.template autoloader-hourly-cronjob.yaml
  - |-
    sed -i -e 's/{{FREQUENCY}}/hourly/'
        -e 's/{{CRON_SCHEDULE}}/0 *\/3 * * */'
        -e 's/{{VERSION}}/v1/'
        -e 's/{{LOAD_PERIOD}}/daily/'
        autoloader-hourly-cronjob.yaml
  # The "weekly" job runs once per month and loads the data from the last month
  # to the last day.
  - cp k8s/autoloader-cronjob.yaml.template autoloader-weekly-cronjob.yaml
  - |-
    sed -i -e 's/{{FREQUENCY}}/weekly/'
        -e 's/{{CRON_SCHEDULE}}/0 0 * * 0/'
        -e 's/{{VERSION}}/v1/'
        -e 's/{{LOAD_PERIOD}}/monthly/'
        autoloader-weekly-cronjob.yaml
  # The "monthly" job runs once per month and loads the data from the beginning of
  # the archive to the last month.
  - cp k8s/autoloader-cronjob.yaml.template autoloader-monthly-cronjob.yaml
  - |-
    sed -i -e 's/{{FREQUENCY}}/monthly/'
        -e 's/{{CRON_SCHEDULE}}/0 0 1 * */'
        -e 's/{{VERSION}}/v1/'
        -e 's/{{LOAD_PERIOD}}/annually/'
        autoloader-monthly-cronjob.yaml
  # Apply
  - kubectl apply -f autoloader-hourly-cronjob.yaml
  - kubectl apply -f autoloader-weekly-cronjob.yaml
  - kubectl apply -f autoloader-monthly-cronjob.yaml
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti

###################
## Autoloader v2
###################
- name: gcr.io/$PROJECT_ID/gcloud-jsonnet-cbif:1.1
  id: "v2/Setup kubectl credentials for autoloader"
  args:
  - gcloud container clusters get-credentials
    --project="${PROJECT_ID}" --region="$_CLUSTER_REGION" "autojoin"
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-autojoin

- name: gcr.io/$PROJECT_ID/gcloud-jsonnet-cbif:1.1
  id: "v2/Deploy the autoloader"
  args:
  - |-
    sed -i -e 's/{{IMAGE_TAG}}/'${COMMIT_SHA}'/'
           -e 's/{{PROJECT_ID}}/'${PROJECT_ID}'/'
           -e 's/{{BQ_PROJECT}}/'${PROJECT_ID}'/'
           -e 's/{{VIEW_PROJECT}}/'$_VIEW_PROJECT'/'
           -e 's/{{GCS_PROJECT}}/'$_GCS_PROJECT'/'
           -e 's/{{MLAB_BUCKET}}/'$_MLAB_BUCKET'/'
           -e 's/{{BUCKETS}}/'$_BUCKETS'/'
           k8s/autoloader.yaml
  - kubectl apply -f k8s/autoloader.yaml
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-autojoin

- name: gcr.io/$PROJECT_ID/gcloud-jsonnet-cbif:1.1
  id: "v2/Deploy cron jobs"
  args:
  # The "hourly" job runs every 3 hours and loads the data for the last day.
  - cp k8s/autoloader-cronjob.yaml.template autoloader-hourly-cronjob.yaml
  - |-
    sed -i -e 's/{{FREQUENCY}}/hourly/'
        -e 's/{{CRON_SCHEDULE}}/0 *\/3 * * */'
        -e 's/{{VERSION}}/v2/'
        -e 's/{{LOAD_PERIOD}}/daily/'
        autoloader-hourly-cronjob.yaml
  # The "weekly" job runs once per month and loads the data from the last month
  # to the last day.
  - cp k8s/autoloader-cronjob.yaml.template autoloader-weekly-cronjob.yaml
  - |-
    sed -i -e 's/{{FREQUENCY}}/weekly/'
        -e 's/{{CRON_SCHEDULE}}/0 0 * * 0/'
        -e 's/{{VERSION}}/v2/'
        -e 's/{{LOAD_PERIOD}}/monthly/'
        autoloader-weekly-cronjob.yaml
  # The "monthly" job runs once per month and loads the data from the beginning of
  # the archive to the last month.
  - cp k8s/autoloader-cronjob.yaml.template autoloader-monthly-cronjob.yaml
  - |-
    sed -i -e 's/{{FREQUENCY}}/monthly/'
        -e 's/{{CRON_SCHEDULE}}/0 0 1 * */'
        -e 's/{{VERSION}}/v2/'
        -e 's/{{LOAD_PERIOD}}/annually/'
        autoloader-monthly-cronjob.yaml
  # Apply
  - kubectl apply -f autoloader-hourly-cronjob.yaml
  - kubectl apply -f autoloader-weekly-cronjob.yaml
  - kubectl apply -f autoloader-monthly-cronjob.yaml
  env:
  - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-autojoin