Build Observatory API Image & Push to Google Cloud Artifact Registry (#558)

Author: jdddog

.github/workflows/build-api-container-develop.yml

@@ -0,0 +1,62 @@
+name: Build Observatory API
+
+# Run workflow on push to any branch or tag
+on: [ push ]
+
+env:
+  REGISTRY: us-docker.pkg.dev
+  USER_NAME: _json_key_base64
+  IMAGE_PATH: observatory-platform/observatory-api
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    strategy:
+      matrix:
+        environment: [ develop, staging, production ]
+        include:
+          - environment: develop
+            gcp_project_id: DEVELOP_GCP_PROJECT_ID
+          - environment: staging
+            gcp_project_id: STAGING_GCP_PROJECT_ID
+          - environment: production
+            gcp_project_id: PRODUCTION_GCP_PROJECT_ID
+
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USER_NAME }}
+          password: ${{ secrets.ARTIFACT_REGISTRY_GCP_SERVICE_KEY }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ secrets[matrix.gcp_project_id] }}/${{ env.IMAGE_PATH }}
+          tags: |
+            # On branch
+            type=ref,event=branch
+
+            # Set latest tag for main branch
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+            # On push tag use version
+            type=semver,pattern={{version}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./observatory-api/
+          push: true
+
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/build-api-container-production.yml

@@ -30,8 +30,12 @@ ENV PYTHONUNBUFFERED True
 RUN mkdir /observatory-api
 COPY . ./observatory-api
 
+# Setup correct requirements
+WORKDIR /observatory-api
+RUN mv -f requirements.cloudrun.txt requirements.txt
+
 # Install observatory-api
-RUN pip3 install /observatory-api
+RUN pip3 install .
 
 # Install berglas
 COPY --from=gcr.io/berglas/berglas:latest /bin/berglas /bin/berglas

.github/workflows/build-api-container-staging.yml

@@ -0,0 +1,13 @@
+gunicorn>=20.1.0,<21
+Flask>=1.1.4,<2
+connexion[swagger-ui]>=2.7.0,<3
+elasticsearch>=7.13.3,<8
+pyyaml>=5.1,<6
+SQLAlchemy>=1.3.18,<2
+psycopg2-binary>=2.9.1,<3
+urllib3>=1.25.11,<2
+python-dateutil>=2.8.2,<3
+certifi>=2021.5.30
+pendulum>=2.1.2,<3
+openapi-spec-validator>=0.3.1,<1
+markupsafe==2.0.1

.github/workflows/build-observatory-api.yml

@@ -1011,7 +1011,7 @@ def config_api(config: TerraformConfig):
 
         click.echo("Configuring the Observatory API")
 
-        text = "Custom domain name for the API, used for the google cloud endpoints service"
+        text = "Custom domain name for the API, used for the Google Cloud Endpoints service"
         default = "api.observatory.academy"
         domain_name = click.prompt(text=text, type=str, default=default, show_default=True)
 
@@ -1020,7 +1020,17 @@ def config_api(config: TerraformConfig):
         choices = click.Choice(choices=["project_id", "environment"], case_sensitive=False)
         subdomain = click.prompt(text=text, type=choices, default=default, show_default=True, show_choices=True)
 
+        text = "Observatory API Docker Image"
+        default = "us-docker.pkg.dev/academic-observatory/observatory-platform/observatory-api:latest"
+        api_image = click.prompt(text=text, type=str, default=default, show_default=True)
+
+        text = "Endpoints Runtime Docker Image"
+        default = "gcr.io/endpoints-release/endpoints-runtime-serverless:2"
+        er_image = click.prompt(text=text, type=str, default=default, show_default=True)
+
         config.api = Api(
             domain_name=domain_name,
             subdomain=subdomain,
+            api_image=api_image,
+            er_image=er_image,
         )

observatory-api/Dockerfile

@@ -29,20 +29,18 @@
 import yaml
 from cerberus import Validator
 from cryptography.fernet import Fernet
+
 from observatory.platform.cli.click_utils import (
     INDENT1,
-    INDENT2,
     INDENT3,
     comment,
     indent,
 )
 from observatory.platform.terraform_api import TerraformVariable
 from observatory.platform.utils.airflow_utils import AirflowVars
-from observatory.platform.utils.config_utils import module_file_path
 from observatory.platform.utils.config_utils import (
     observatory_home as default_observatory_home,
 )
-from observatory.platform.utils.jinja2_utils import render_template
 
 
 def generate_fernet_key() -> str:
@@ -601,13 +599,24 @@ class Api:
         domain_name: the custom domain name of the API
         subdomain: the subdomain of the API, can be either based on the google project id or the environment. When
         based on the environment, there is no subdomain for the production environment.
+        api_image: the path to the Observatory API image on Google Cloud Artifact Registry.
+        er_image: the path to the Google Cloud Endpoints Runtime image.
     """
 
     domain_name: str
     subdomain: str
+    api_image: str
+    er_image: str = "gcr.io/endpoints-release/endpoints-runtime-serverless:2"
 
     def to_hcl(self):
-        return to_hcl({"domain_name": self.domain_name, "subdomain": self.subdomain})
+        return to_hcl(
+            {
+                "domain_name": self.domain_name,
+                "subdomain": self.subdomain,
+                "api_image": self.api_image,
+                "er_image": self.er_image,
+            }
+        )
 
     @staticmethod
     def from_dict(dict_: Dict) -> Api:
@@ -619,7 +628,9 @@ def from_dict(dict_: Dict) -> Api:
 
         domain_name = dict_.get("domain_name")
         subdomain = dict_.get("subdomain")
-        return Api(domain_name, subdomain)
+        api_image = dict_.get("api_image")
+        er_image = dict_.get("er_image", Api.er_image)
+        return Api(domain_name, subdomain, api_image, er_image=er_image)
 
 
 def is_base64(text: bytes) -> bool:
@@ -1554,6 +1565,8 @@ def make_schema(backend_type: BackendType) -> Dict:
             "schema": {
                 "domain_name": {"required": True, "type": "string"},
                 "subdomain": {"required": True, "type": "string", "allowed": ["project_id", "environment"]},
+                "api_image": {"required": True, "type": "string"},
+                "er_image": {"required": False, "type": "string"},
             },
         }
 
@@ -1866,12 +1879,15 @@ def api(api: Api) -> List[str]:
             api = Api(
                 domain_name="api.observatory.academy",
                 subdomain="project_id",
+                api_image="us-docker.pkg.dev/gcp-project-id/observatory-platform/observatory-api:latest",
             )
 
         lines = [
             "api:\n",
             indent(f"domain_name: {api.domain_name}\n", INDENT1),
             indent(f"subdomain: {api.subdomain}\n", INDENT1),
+            indent(f"api_image: {api.api_image}\n", INDENT1),
+            indent(f"er_image: {api.er_image}\n", INDENT1),
         ]
 
         return lines

observatory-api/requirements.cloudrun.txt

@@ -54,23 +54,14 @@ module "observatory_db_uri" {
   service_account_email = google_service_account.api-backend_service_account.email
 }
 
-# Create data resource to keep track of latest image change
-data "archive_file" "build_image_info"{
-  # the only available type
-  type = "zip"
-  source_file = "./api_image_build.txt"
-  output_path = "./api_image_build.zip"
-}
-
-
 resource "google_cloud_run_service" "api_backend" {
-  name     = "api-backend"
+  name = "api-backend"
   location = var.google_cloud.region
 
   template {
     spec {
       containers {
-        image = "gcr.io/${var.google_cloud.project_id}/observatory-api"
+        image = var.api.api_image
         env {
           name = "ES_API_KEY"
           value = "sm://${var.google_cloud.project_id}/elasticsearch-api_key"
@@ -89,8 +80,6 @@ resource "google_cloud_run_service" "api_backend" {
     metadata {
       annotations = {
         "autoscaling.knative.dev/maxScale" = "10"
-        # make resource dependent on sha256 of file describing image info
-        build_image = data.archive_file.build_image_info.output_base64sha256
         "run.googleapis.com/vpc-access-egress" : "private-ranges-only"
         "run.googleapis.com/vpc-access-connector" = "projects/${var.google_cloud.project_id}/locations/${var.google_cloud.region}/connectors/${var.vpc_connector_name}"
       }
@@ -151,15 +140,14 @@ resource "google_project_iam_member" "api-gateway_service_account_servicecontrol
   member = "serviceAccount:${google_service_account.api-gateway_service_account.email}"
 }
 
-# Create/update Cloud Run service
 resource "google_cloud_run_service" "api_gateway" {
   name = "api-gateway"
   location = var.google_cloud.region
   project = var.google_cloud.project_id
   template {
     spec {
       containers {
-        image = "gcr.io/endpoints-release/endpoints-runtime-serverless:2"
+        image = var.api.er_image
         env {
           name = "ENDPOINTS_SERVICE_NAME"
           value = google_endpoints_service.api.service_name

observatory-platform/observatory/platform/cli/generate_command.py

@@ -4,10 +4,14 @@ Settings related to the Observatory API
 
 domain_name: the custom domain name for the API, used for the google cloud endpoints service
 subdomain: can be either 'project_id' or 'environment', used to determine a prefix for the domain_name
+api_image: the path to the Observatory API image on Google Cloud Artifact Registry.
+er_image: the path to the Google Cloud Endpoints Runtime image.
 EOF
   type = object({
     domain_name = string
     subdomain = string
+    api_image = string
+    er_image = string
   })
 }