diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..5de1fb8a71 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,13 @@ +{ + "name": "Wally", + "image": "ghcr.io/openhwgroup/wally:latest-devel", + "containerEnv": { + "WALLY": "${containerWorkspaceFolder}" + }, + "customizations": { + "vscode": { + "extensions": [] + } + }, + "postStartCommand": "/etc/wally/container-setup.sh" +} \ No newline at end of file diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml new file mode 100644 index 0000000000..93d9985993 --- /dev/null +++ b/.github/workflows/container-release.yml @@ -0,0 +1,197 @@ +name: Container Release +on: + push: + paths: + - bin/wally-tool-chain-install.sh + - bin/wally-environment-check.sh + - bin/wally-package-install.sh + - bin/installation/** + - containers/wally-devel/** + - .github/workflows/container-release.yml # Self-trigger + workflow_dispatch: + inputs: + publish: + description: "Publish to the container registry" + required: false + default: false + type: boolean + +env: + REGISTRY: ghcr.io/doganulus + IMAGE_NAME: wally + HOSTARCH: amd64 + PLATFORM: linux-amd64 + CONTAINER_VERSION: latest + CONTAINER_LATEST_VERSION: latest + CONTAINERS_ROOT: /home/runner/.local/share/containers + TMPDIR: /home/runner/.local/share/containers/tmp + +permissions: + contents: read + packages: write + +jobs: + buildah-build: + name: Build container images + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + cancel-in-progress: true + + steps: + - name: Install container tools + run: sudo apt-get install podman buildah jq + + - name: Maximize build space + uses: easimon/maximize-build-space@v10 + with: + root-reserve-mb: 2048 # Reserve disk space for repository + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + remove-docker-images: "true" + build-mount-path: ${{ env.CONTAINERS_ROOT }} # The remaining space only for container build + + - run: mkdir -p $TMPDIR + + - name: Prepare environment variables + run: | + echo "HOSTARCH=$(podman info --format='{{.Host.Arch}}')" >> $GITHUB_ENV + echo "PLATFORM=$(podman info --format='{{.Version.OsArch}}' | sed 's/\//-/g')" >> $GITHUB_ENV + echo "CONTAINER_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the GitHub Container registry + uses: redhat-actions/podman-login@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build builder container image + id: build-builder + uses: redhat-actions/buildah-build@v2 + with: + context: . + image: wally + tags: | + ${{ env.CONTAINER_VERSION }}-builder + ${{ env.CONTAINER_VERSION }}-builder-${{ env.HOSTARCH }} + layers: true + oci: true + build-args: | + CONTAINER_VERSION=${{ env.CONTAINER_VERSION }} + CONTAINER_IMAGE_REGISTRY_REMOTE=${{ env.REGISTRY }} + extra-args: | + --target wally-builder + containerfiles: | + ./containers/wally-devel/Dockerfile + + - name: Build devel container image + id: build-devel + uses: redhat-actions/buildah-build@v2 + with: + context: . + image: wally + tags: ${{ env.CONTAINER_VERSION }}-devel ${{ env.CONTAINER_VERSION }}-devel-${{ env.HOSTARCH }} + layers: true + oci: true + build-args: | + CONTAINER_VERSION=${{ env.CONTAINER_VERSION }} + CONTAINER_IMAGE_REGISTRY_REMOTE=${{ env.REGISTRY }} + extra-args: | + --target wally-devel + containerfiles: | + ./containers/wally-devel/Dockerfile + + - name: Push to GitHub Container Repository + if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish + id: push-builder-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + registry: ${{ env.REGISTRY }} + image: ${{ steps.build-builder.outputs.image }} + tags: ${{ env.CONTAINER_VERSION }}-builder-${{ env.HOSTARCH }} + digestfile: ${{ runner.temp }}/digest-wally-builder-${{ env.CONTAINER_VERSION }}-${{ env.PLATFORM }} + + - name: Push to GitHub Container Repository + if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish + id: push-devel-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + registry: ${{ env.REGISTRY }} + image: ${{ steps.build-devel.outputs.image }} + tags: ${{ env.CONTAINER_VERSION }}-devel-${{ env.HOSTARCH }} + digestfile: ${{ runner.temp }}/digest-wally-devel-${{ env.CONTAINER_VERSION }}-${{ env.PLATFORM }} + + - name: Upload digests + if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish + uses: actions/upload-artifact@v4 + with: + name: digest-wally-${{ env.CONTAINER_VERSION }}-${{ env.PLATFORM }} + path: ${{ runner.temp }}/digest-* + if-no-files-found: error + retention-days: 1 + compression-level: 0 # no compression + + buildah-merge: + name: Merge container images + runs-on: ubuntu-24.04 + needs: buildah-build + if: always() && github.event_name == 'workflow_dispatch' && github.event.inputs.publish + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digest-* + merge-multiple: true + + - name: Log in to the GitHub Container registry + uses: redhat-actions/podman-login@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare environment variables + run: | + if [ "${{ env.CONTAINER_VERSION }}" == "latest" ]; then + echo "CONTAINER_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV + fi + echo "CONTAINER_LATEST_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - name: Create and push manifest list for wally-builder + run: | + MANIFEST=wally-builder + FULL_IMAGE_NAME=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + buildah manifest create $MANIFEST + for digest in ${{ runner.temp }}/digests/digest-wally-builder-*; do + echo "Adding $(cat $digest)" + buildah manifest add $MANIFEST $FULL_IMAGE_NAME@$(cat $digest) + done + buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:${{ env.CONTAINER_VERSION }}-builder + if [ "${{ env.CONTAINER_VERSION }}" == "${{ env.CONTAINER_LATEST_VERSION }}" ]; then + buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:latest-builder + fi + + - name: Create and push manifest list for wally-devel + run: | + MANIFEST=wally-devel + FULL_IMAGE_NAME=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + buildah manifest create $MANIFEST + for digest in ${{ runner.temp }}/digests/digest-wally-devel-*; do + echo "Adding $(cat $digest)" + buildah manifest add $MANIFEST $FULL_IMAGE_NAME@$(cat $digest) + done + buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:${{ env.CONTAINER_VERSION }}-devel + if [ "${{ env.CONTAINER_VERSION }}" == "${{ env.CONTAINER_LATEST_VERSION }}" ]; then + buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:latest-devel + fi diff --git a/.github/workflows/container-retention.yml b/.github/workflows/container-retention.yml new file mode 100644 index 0000000000..5bcc770cc0 --- /dev/null +++ b/.github/workflows/container-retention.yml @@ -0,0 +1,24 @@ +name: Container Retention +on: + workflow_dispatch: + schedule: + - cron: "0 12 * * 0" + +permissions: + contents: read + packages: write + +jobs: + clean: + runs-on: ubuntu-latest + name: Github Container Registry Retention Policy + steps: + - name: Clean up untagged images + uses: snok/container-retention-policy@v3.0.0 + with: + account: openhwgroup + token: ${{ secrets.GITHUB_TOKEN }} + image-names: "wally" + tag-selection: untagged + cut-off: 1h + dry-run: false diff --git a/bin/wally-environment-check.sh b/bin/wally-environment-check.sh index eb1a297db6..c426f183a1 100755 --- a/bin/wally-environment-check.sh +++ b/bin/wally-environment-check.sh @@ -160,12 +160,14 @@ while [[ "$#" -gt 0 ]]; do case $1 in -c|--clean) clean=true ;; --no-buildroot) no_buildroot=true ;; + --no-skywater) no_skywater=true ;; --packages-only) packages_only=true;; -h|--help) echo -e "Usage: $0 [\$RISCV] [options]" echo -e "${BOLD}Options:${ENDC}" echo -e " -c, --clean Remove build directories after installation" echo -e " --no-buildroot Skip installing Buildroot and Linux testvectors" + echo -e " --no-skywater Skip installing Skywater PDK" echo -e " \$RISCV Directory to install RISC-V tools (default: /opt/riscv as root, $HOME/riscv otherwise)" exit 0 ;; *) RISCV="$1" ;; diff --git a/bin/wally-tool-chain-install.sh b/bin/wally-tool-chain-install.sh index fee173f92d..c49f361744 100755 --- a/bin/wally-tool-chain-install.sh +++ b/bin/wally-tool-chain-install.sh @@ -107,8 +107,11 @@ source "$WALLY"/bin/installation/verilator-install.sh # OSU Skywater 130 cell library (https://foss-eda-tools.googlesource.com/skywater-pdk/libs/sky130_osu_sc_t12) # The OSU Skywater 130 cell library is a standard cell library that is used to synthesize Wally. -source "$WALLY"/bin/installation/skywater-lib-install.sh - +if [ "$no_skywater" = true ]; then + echo -e "${OK_COLOR}Skipping OSU Skywater 130 cell library installation.${ENDC}" +else + source "$WALLY"/bin/installation/skywater-lib-install.sh +fi # Buildroot and Linux testvectors # Buildroot is used to boot a minimal version of Linux on Wally. diff --git a/containers/Makefile b/containers/Makefile new file mode 100644 index 0000000000..8c57837d18 --- /dev/null +++ b/containers/Makefile @@ -0,0 +1,40 @@ +WALLY_CONTAINER_BASE_IMAGE ?= docker.io/almalinux/9-base:9.6 +WALLY_CONTAINER_IMAGE_REGISTRY ?= localhost +WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE ?= ghcr.io/openhwgroup +WALLY_CONTAINER_IMAGE_NAME ?= wally +WALLY_CONTAINER_TARGET_VERSION ?= latest +WALLY_CONTAINER_CURRENT_VERSION ?= $(shell date +'%Y%m%d') + +WALLY_CONTAINER_BUILD_CONTEXT ?= .. # Set to the project root + +info: + @echo "Container version: ${WALLY_CONTAINER_TARGET_VERSION}" + @echo "Container current version: ${WALLY_CONTAINER_CURRENT_VERSION}" + +builder: + buildah build \ + -f wally-devel/Dockerfile \ + --build-arg WALLY_CONTAINER_BASE_IMAGE=${WALLY_CONTAINER_BASE_IMAGE} \ + --build-arg WALLY_CONTAINER_VERSION=${WALLY_CONTAINER_CURRENT_VERSION} \ + --format oci \ + --layers=true \ + --target wally-builder \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-builder \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-builder \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-builder \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-builder \ + ${WALLY_CONTAINER_BUILD_CONTEXT} + +devel: builder + buildah build \ + -f wally-devel/Dockerfile \ + --build-arg WALLY_CONTAINER_BASE_IMAGE=${WALLY_CONTAINER_BASE_IMAGE} \ + --build-arg WALLY_CONTAINER_VERSION=${WALLY_CONTAINER_CURRENT_VERSION} \ + --format oci \ + --layers=true \ + --target wally-devel \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-devel \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-devel \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-devel \ + --tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-devel \ + ${WALLY_CONTAINER_BUILD_CONTEXT} diff --git a/containers/wally-devel/Dockerfile b/containers/wally-devel/Dockerfile new file mode 100644 index 0000000000..f510ba68b6 --- /dev/null +++ b/containers/wally-devel/Dockerfile @@ -0,0 +1,41 @@ +ARG WALLY_CONTAINER_VERSION=latest +ARG WALLY_CONTAINER_BASE_IMAGE=docker.io/almalinux/9-base:9.6 + +ARG WALLY_CONTAINER_BUILDER_IMAGE=wally-builder +ARG WALLY_CONTAINER_DEVEL_IMAGE=wally-devel + +ARG RISCV_GNU_TOOLCHAIN_VERSION=23863c2ca74e6c050f0c97e7af61f5f1776aadd1 + +FROM ${WALLY_CONTAINER_BASE_IMAGE} as wally-builder +ARG TARGETARCH TARGETOS TARGETPLATFORM TARGETVARIANT + +ARG RISCV_GNU_TOOLCHAIN_VERSION +ENV RISCV_GNU_TOOLCHAIN_VERSION=${RISCV_GNU_TOOLCHAIN_VERSION} +ENV RISCV_INSTALL_PREFIX=/opt/riscv + +ENV WALLY_HOME=/root +ENV WALLY_SOURCE_DIR=/workspaces/cvw +ENV WALLY_PYTHON_VENV_DIR=${WALLY_HOME}/.venv/wally +ENV WALLY_PYTHON_EXECUTABLE=${WALLY_PYTHON_VENV_DIR}/bin/python + +ENV RISCV=${RISCV_INSTALL_PREFIX} +ENV WALLY=${WALLY_SOURCE_DIR} + +RUN --mount=type=bind,source=./bin,target=${WALLY_SOURCE_DIR}/bin \ + bash "${WALLY_SOURCE_DIR}/bin/wally-package-install.sh" --clean + +RUN --mount=type=bind,source=./bin,target=${WALLY_SOURCE_DIR}/bin \ + bash "${WALLY_SOURCE_DIR}/bin/installation/riscv-gnu-toolchain-install.sh" --clean + +RUN --mount=type=bind,source=./bin,target=${WALLY_SOURCE_DIR}/bin \ + --mount=type=bind,source=./linux,target=${WALLY_SOURCE_DIR}/linux \ + bash "${WALLY_SOURCE_DIR}/bin/wally-tool-chain-install.sh" --clean --no-buildroot --no-skywater + +COPY --chmod=755 ./containers/wally-devel/container-setup.sh /etc/wally/container-setup.sh + +ENV WALLY_CORE_DUMP_SIZE=300000 +ENV WALLY_PYTHON_VENV_DIR=${RISCV_INSTALL_PREFIX}/riscv-python +ENV WALLY_PYTHON_EXECUTABLE=${WALLY_PYTHON_VENV_DIR}/bin/python + +FROM ${WALLY_CONTAINER_BUILDER_IMAGE} as wally-devel +ARG TARGETARCH TARGETOS TARGETPLATFORM TARGETVARIANT diff --git a/containers/wally-devel/container-setup.sh b/containers/wally-devel/container-setup.sh new file mode 100644 index 0000000000..dd5566d3b1 --- /dev/null +++ b/containers/wally-devel/container-setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +# Append PATH and source command to ~/.bashrc +echo 'export PATH="${WALLY}/bin:${RISCV}/bin:$PATH"' >> "$HOME/.bashrc" +echo 'export CVW_ARCH_VERIF="${WALLY}/addins/cvw-arch-verif"' >> "$HOME/.bashrc" + +echo 'source ${WALLY_PYTHON_VENV_DIR}/bin/activate' >> "$HOME/.bashrc" + +# Set core dump size (default 300000 KB if not provided) +ulimit -c "${WALLY_CORE_DUMP_SIZE:=300000}" + +# Install pre-commit hook if missing +if [ ! -e "$WALLY/.git/hooks/pre-commit" ]; then + pushd "$WALLY" > /dev/null || exit 1 + echo "Installing pre-commit hooks" + pre-commit install + popd > /dev/null || exit 1 +fi \ No newline at end of file