Skip to content

Fixes for the pipeline #3731

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions .envrc
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,9 @@ then
# for more details
fi

local bbl_state_dir=../app-autoscaler-env-bbl-state/bbl-state
if has bbl && [[ -d "$bbl_state_dir" ]]
then
BBL_STATE_DIRECTORY="$(realpath "$bbl_state_dir")"
export BBL_STATE_DIRECTORY
eval "$(bbl print-env)"
fi
local bbl_state_dir=${BBL_STATE_PATH:-'../app-autoscaler-env-bbl-state/bbl-state'}
source './ci/autoscaler/scripts/common.sh'
bosh_login "${bbl_state_dir}"
echo '🚸 Login for bosh and concourse successful! For more execute: `make cf-login`'

export DBURL="postgres://postgres:postgres@localhost/autoscaler"
export DBURL='postgres://postgres:postgres@localhost/autoscaler'
59 changes: 48 additions & 11 deletions .github/workflows/image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,55 @@ on:
paths:
- 'ci/dockerfiles/**'
- '.github/workflows/image.yml'
- '.tool-versions'
pull_request:
paths:
- 'ci/dockerfiles/**'
- '.github/workflows/image.yml'
- '.tool-versions'
workflow_dispatch:

env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}

jobs:
build-and-push-devbox-image:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
name: Build and Push app-autoscaler-release-devbox
env:
IMAGE_NAME: ${{ github.repository }}-devbox
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

- name: Build and push
id: build-and-push
uses: docker/build-push-action@v5
with:
context: .
file: ci/dockerfiles/autoscaler-devbox/Dockerfile
push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main

build-and-push-image:
runs-on: ubuntu-latest
container: jetpackio/devbox:latest@sha256:3d164648f5ca0fa66cd080ee7c895af54e0a3ec19e19b232a2fea21628e82cfb
permissions:
contents: read
packages: write
Expand All @@ -29,40 +64,42 @@ jobs:
matrix:
image_suffix: ["tools"]
name: Build and Push app-autoscaler-release-${{ matrix.image_suffix }}
env:
IMAGE_NAME: ${{ github.repository }}
steps:
- name: Checkout repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image_suffix }}

- name: Get ginkgo version from autoscaler-release
id: ginkgo
run: |
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 [actionlint] reported by reviewdog 🐶
shellcheck reported issue in this script: SC2028:info:2:6: echo may not expand escape sequences. Use printf [shellcheck]

version=$(devbox info ginkgo | head --lines=1 | cut --field=2 --delimiter=" ")
echo "GINKGO version from devbox: '${version}'"
version=$(grep "ginkgo " .tool-versions| cut -f 2 -d " ")
echo "GINKGO version from .tool-versions: \'${version}\'"
echo "version=${version}" >> "$GITHUB_OUTPUT"

- id: get-golang-version
shell: bash
run: |
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 [actionlint] reported by reviewdog 🐶
shellcheck reported issue in this script: SC2028:info:2:6: echo may not expand escape sequences. Use printf [shellcheck]

version=$(devbox info go | head --lines=1 | cut --field=2 --delimiter=" ")
echo "Go version from devbox: '${version}'"
version=$(grep "golang " .tool-versions| cut -f 2 -d " ")
echo "Go version from .tool-versions: \'${version}\'"
echo "version=${version}" >> "$GITHUB_OUTPUT"

- name: Build and push
id: build-and-push
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6
uses: docker/build-push-action@v5
with:
context: ci/dockerfiles/autoscaler-${{ matrix.image_suffix }}
push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
Expand Down
18 changes: 12 additions & 6 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,14 @@
# # Edit: Please use global git-ignore for this, see official hints in the description of gitignore
# # on <https://git-scm.com/docs/gitignore>.
#
# # macOS system file that stores folder metadata (safe to ignore)
# .DS_Store


# ================================================================================
# 🤖 Generated files
# ================================================================================

config/dev.yml
config/private.yml
releases/*.tgz
Expand Down Expand Up @@ -66,16 +77,11 @@ keys/
.trunk


# ==================== 📦 Devbox ====================
# ==================== 📦 Devbox, ❄ Nix, direnv ====================
.devbox

# ==================== ❄ Nix ====================

# Ignore links to build-output:
/result
/result-*

.direnv

# # macOS system file that stores folder metadata (safe to ignore)
.DS_Store
23 changes: 10 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,11 @@ spec-test:
bundle exec rspec

.PHONY: bosh-release
bosh-release: go-mod-tidy go-mod-vendor scheduler db build/autoscaler-test.tgz
build/autoscaler-test.tgz:
bosh-release: build/autoscaler-test.tgz
# 🚸 In the next line, the order of the dependencies is important. Generated code needs to be
# already there for `go-mod-tidy` to work. See additional comment for that target in
# ./src/autoscaler/Makefile.
build/autoscaler-test.tgz: build_all go-mod-tidy go-mod-vendor
@echo " - building bosh release into build/autoscaler-test.tgz"
@mkdir -p build
@bosh create-release --force --timestamp-version --tarball=build/autoscaler-test.tgz
Expand Down Expand Up @@ -442,8 +445,11 @@ cleanup-autoscaler-deployments:
@${CI_DIR}/autoscaler/scripts/cleanup-autoscaler-deployments.sh

.PHONY: cf-login
cf-login: ## Login to OSS CF dev environment
@${CI_DIR}/autoscaler/scripts/cf-login.sh
cf-login:
@echo '⚠️ Please note that this login only works for cf and concourse,' \
'in spite of performing a login as well on bosh and credhub.' \
'The necessary changes to the environment get lost when make exits its process.'
@${CI_DIR}/autoscaler/scripts/os-infrastructure-login.sh

.PHONY: setup-performance
setup-performance: build-test-app
Expand Down Expand Up @@ -484,15 +490,6 @@ alerts-silence:
${CI_DIR}/autoscaler/scripts/silence_prometheus_alert.sh BOSHJobEphemeralDiskPredictWillFill ;\
${CI_DIR}/autoscaler/scripts/silence_prometheus_alert.sh BOSHJobUnhealthy ;

.PHONY: docker-login docker docker-image
docker-login: target/docker-login
target/docker-login:
docker login ghcr.io
@touch $@
docker-image: docker-login
docker build -t ghcr.io/cloudfoundry/app-autoscaler-release-tools:latest ci/dockerfiles/autoscaler-tools
docker push ghcr.io/cloudfoundry/app-autoscaler-release-tools:latest


validate-openapi-specs: $(wildcard ./api/*.openapi.yaml)
for file in $^ ; do \
Expand Down
3 changes: 3 additions & 0 deletions ci/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# ================================================================================
# 🤖 Generated files
# ================================================================================
39 changes: 39 additions & 0 deletions ci/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
FROM jetpackio/devbox:latest

# Installing your devbox project
WORKDIR /code
USER root:root
RUN mkdir --parents /code && chown ${DEVBOX_USER}:${DEVBOX_USER} /code
USER ${DEVBOX_USER}:${DEVBOX_USER}
COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.json devbox.json
COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} devbox.lock devbox.lock


# # 💡 Here maybe the chown is missing!
# # Step 6: Copying local flakes directories
#COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} ./.devbox/virtenv/mysql/flake ./.devbox/virtenv/mysql/flake
COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} ./.devbox ./.devbox
COPY --chown=${DEVBOX_USER}:${DEVBOX_USER} ./nix ./nix

# 💡 The issue probably is, that we don't copy the necessary files before.
# I alternatively will try to avoid the copies and read-only-mount those directories
# into the image.
RUN \
# --mount=type=bind,source=./nix,target=/code/nix,readonly \
# --mount=type=bind,source=./.devbox,target=/code/.devbox,readonly \
<<-EODI
echo "ls -lah /code: $(ls -lah /code)"
echo "ls -lah /code/.devbox: $(ls -lah /code/.devbox)"
echo "ls -lah /code/.devbox/gen: $(ls -lah /code/.devbox/gen)"
devbox run -- echo "Installed Packages."
EODI

# Make use of the installed devbox-environment:
ENV \
PATH="/code/.devbox/nix/profile/default/bin:${PATH}" \
LD_LIBRARY_PATH="/code/.devbox/nix/profile/default/lib"

# 🚧 To-do: Check if “ENTRYPOINT” is not the better alternative, see:
# <https://www.bmc.com/blogs/docker-cmd-vs-entrypoint>
ENTRYPOINT ["devbox", "run", "--"]
# CMD ["devbox", "shell"]
13 changes: 13 additions & 0 deletions ci/Dockerfile.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# ==================== Ignore everything by default ====================
*



# ==================== What we actually need ====================
!./nix
!/flake.nix
!/flake.lock

!./.devbox
!/devbox.json
!/devbox.lock
66 changes: 62 additions & 4 deletions ci/Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,65 @@
.ONESHELL:
SHELL := /bin/bash
.SHELLFLAGS := -eu -o pipefail -c

repository-root := $(shell realpath --relative-to $$(pwd) $$(git rev-parse --show-toplevel))

# All files that together define the development-environment:
#
# 🚧 To-do: Check if each of the last two lines should be substituted by something like:
# `$(wildcard ./.devbox/**/flake.{nix,lock})`
development-environment-definition := \
${repository-root}/nix/flake.nix \
${repository-root}/nix/flake.lock \
${repository-root}/devbox.json \
${repository-root}/devbox.lock \
${repository-root}/.devbox/virtenv/mysql/flake/flake.nix \
${repository-root}/.devbox/virtenv/mysql/flake/flake.lock
# 🚸 Please be aware that devbox automatically creates
# '${repository-root}/.devbox/virtenv/mysql/flake' when it loads the development-environment.

# The generated Dockerfile just depends on the content of the used “devbox.json” and its lockfile.
# It is marked secondary to not auto-clean it.
#
# ⚠️ As the Dockerfile that is auto-generated via its target “Dockerfile” is not of high quality and
# may contain errors, its dependencies are commented out to prevent re-generation by gnu-make.
.SECONDARY: Dockerfile
Dockerfile: # ./Makefile ${repository-root}/devbox.json ${repository-root}/devbox.lock
devbox generate dockerfile --config='${repository-root}'
mv '${repository-root}/Dockerfile' .

# 🚸 Overwrite the following parameters to work with a different repository on a different registry:
DOCKER_REGISTRY ?= ghcr.io
DOCKER_REPOSITORY ?= cloudfoundry/app-autoscaler-release-tools

.PHONY: docker-login
docker-login:
@logged_in=$$(jq '.auths | has("${DOCKER_REGISTRY}")' ~/.docker/config.json)
readonly logged_in
if [[ "$${logged_in}" != 'true' ]]
then
docker login '${DOCKER_REGISTRY}'
else
echo 'Already logged in into ${DOCKER_REGISTRY}, skipping login.'
fi

oci-image-location := ${repository-root}/build/app-autoscaler-release-tools.tar.zst
.PHONY: oci-image docker-image
oci-image: ${oci-image-location}
docker-image: ${oci-image-location}

${oci-image-location}: ./Makefile Dockerfile ${development-environment-definition}
@readonly image_tag='${DOCKER_REGISTRY}/${DOCKER_REPOSITORY}:latest'
docker build --file='./Dockerfile' '${repository-root}' --tag="$${image_tag}"
docker image save "$${image_tag}" | zstd > '${oci-image-location}'

.PHONY: oci-release-%
oci-release-%: docker-login docker-image
@ARTIFACT='${DOCKER_REGISTRY}/${DOCKER_REPOSITORY}:$*'
echo "Uploading image" "$${ARTIFACT}"
docker push "$${ARTIFACT}"



.PHONY: set-autoscaler-pipeline
set-autoscaler-pipeline:
Expand All @@ -15,10 +77,6 @@ unpause-pipeline:
delete-pipeline:
@./scripts/delete-pipeline.sh

.PHONY: set-target
set-target:
@fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org

.PHONY: lint
lint:
@shellcheck --external-sources --shell='bash' **/scripts/*.sh
Expand Down
Loading
Loading