Merge pull request #1661 from tlaurion/wip-nix-for-build

Move to nix buildstack (and nix develop produced docker image used under CircleCI)

Author: tlaurion

.circleci/config.yml

@@ -0,0 +1 @@
+has nix && use flake

.envrc

@@ -1,25 +1,25 @@
-.*.sw*
-*.xz
+*.bad
 *.bz2
+*.cpio
+*.dep
+*.ffs
+*.fv
 *.gz
-*.sign
-*.rom
-*.o
-*.gz
-*.tgz
 *.img
-*.rom
-*.cpio
-typescript*
-config/*.old
 *.log
-*~
-crossgcc
-clean
+*.lz
+*.o
+*.rom
 *.sec
-*.dep
-*.ffs
+*.sign
+*.tgz
 *.vol
-*.lz
-*.fv
-*.bad
+*.xz
+*~
+.*.sw*
+/.direnv
+clean
+config/*.old
+crossgcc
+typescript*
+result

.gitignore

@@ -25,11 +25,48 @@ INSTALL		= $(pwd)/install/$(CONFIG_TARGET_ARCH)
 log_dir		= $(build)/log
 board_build	= $(build)/$(BOARD)
 
+
+# Estimated memory required per job in GB (e.g., 1GB for gcc)
+MEM_PER_JOB_GB ?= 1
+
 # Controls how many parallel jobs are invoked in subshells
-CPUS		?= $(shell nproc)
-MAKE_JOBS	?= -j$(CPUS) --max-load 16
+CPUS            ?= $(shell nproc)
+AVAILABLE_MEM_GB   ?= $(shell cat /proc/meminfo | grep MemAvailable | awk '{print int($$2 / 1024)}')
+
+# Calculate the maximum number of jobs based on available memory
+MAX_JOBS_MEM := $(shell echo $$(( $(AVAILABLE_MEM_GB) / $(MEM_PER_JOB_GB) )))
+
+# Use the minimum of the system's CPUs and the calculated max jobs based on memory
+CPUS            := $(shell echo $$(($(CPUS) < $(MAX_JOBS_MEM) ? $(CPUS) : $(MAX_JOBS_MEM))))
+
+# Load average can be adjusted to be higher than CPUS to allow for some CPU overcommit
+# Multiply by 3 and then divide by 2 to achieve the effect of multiplying by 1.5 using integer arithmetic
+LOADAVG         ?= $(shell echo $$(( ($(CPUS) * 3) / 2 )))
+
+# Construct MAKE_JOBS with dynamic CPU count and load average
+MAKE_JOBS       := -j$(CPUS) --load-average=$(LOADAVG) # Add other flags as needed to be more adaptive to CIs
+
+# Print out the settings and compare system values with actual ones used
+$(info ----------------------------------------------------------------------)
+$(info !!!!!! BUILD SYSTEM INFO !!!!!!)
+$(info System CPUS: $(shell nproc))
+$(info System Available Memory: $(AVAILABLE_MEM_GB) GB)
+$(info System Load Average: $(shell uptime | awk '{print $$10}'))
+$(info ----------------------------------------------------------------------)
+$(info Used **CPUS**: $(CPUS))
+$(info Used **LOADAVG**: $(LOADAVG))
+$(info Used **AVAILABLE_MEM_GB**: $(AVAILABLE_MEM_GB) GB)
+$(info ----------------------------------------------------------------------)
+$(info **MAKE_JOBS**: $(MAKE_JOBS))
+$(info )
+$(info Variables available for override (use 'make VAR_NAME=value'):)
+$(info **CPUS** (default: number of processors, e.g., 'make CPUS=4'))
+$(info **LOADAVG** (default: 1.5 times CPUS, e.g., 'make LOADAVG=54'))
+$(info **AVAILABLE_MEM_GB** (default: memory available on the system in GB, e.g., 'make AVAILABLE_MEM_GB=4'))
+$(info **MEM_PER_JOB_GB** (default: 1GB per job, e.g., 'make MEM_PER_JOB_GB=2'))
+$(info ----------------------------------------------------------------------)
+$(info !!!!!! Build starts !!!!!!)
 
-WGET ?= wget
 
 # Timestamps should be in ISO format
 DATE=`date --rfc-3339=seconds`
@@ -162,6 +199,7 @@ heads_cc	:= $(CROSS)gcc \
 	-fdebug-prefix-map=$(pwd)=heads \
 	-gno-record-gcc-switches \
 	-D__MUSL__ \
+	--sysroot  $(INSTALL) \
 	-isystem $(INSTALL)/include \
 	-L$(INSTALL)/lib \
 
@@ -230,12 +268,10 @@ all payload:
 FORCE:
 
 # Copies config while replacing predefined placeholders with actual values
+# This is used in a command like 'this && $(call install_config ...) && that'
+# so it needs to evaluate to a shell command.
 define install_config =
-	sed -e 's!@BOARD_BUILD_DIR@!$(board_build)!g' \
-	    -e 's!@BLOB_DIR@!$(pwd)/blobs!g' \
-	    -e 's!@BRAND_DIR@!$(pwd)/branding/$(BRAND_NAME)!g' \
-	    -e 's!@BRAND_NAME@!$(BRAND_NAME)!g' \
-	    "$1" > "$2"
+	$(pwd)/bin/prepare_module_config.sh "$1" "$2" "$(board_build)" "$(BRAND_NAME)"
 endef
 
 # Make helpers to operate on lists of things
@@ -783,6 +819,8 @@ $(board_build)/$(CB_OUTPUT_BASENAME)-gpg-injected.rom: $(board_build)/$(CB_OUTPU
 	./bin/inject_gpg_key.sh --cbfstool "$(build)/$(coreboot_dir)/cbfstool" \
 		"$(board_build)/$(CB_OUTPUT_FILE_GPG_INJ)" "$(PUBKEY_ASC)"
 
+
+#Dev cycles helpers:
 real.clean:
 	for dir in \
 		$(module_dirs) \
@@ -794,4 +832,28 @@ real.clean:
 	done
 	cd install && rm -rf -- *
 real.gitclean:
+	@echo "Cleaning the repository using Git ignore file as a base..."
+	@echo "This will wipe everything not in the Git tree, but keep downloaded coreboot forks (detected as Git repos)."
 	git clean -fxd
+
+real.gitclean_keep_packages:
+	@echo "Cleaning the repository using Git ignore file as a base..."
+	@echo "This will wipe everything not in the Git tree, but keep the 'packages' directory."
+	git clean -fxd -e "packages"
+
+real.remove_canary_files-extract_patch_rebuild_what_changed:
+	@echo "Removing 'canary' files to force Heads to restart building board configurations..."
+	@echo "This will check package integrity, extract them, redo patching on files, and rebuild what needs to be rebuilt."
+	@echo "It will also reinstall the necessary files under './install'."
+	@echo "Limitations: If a patch creates a file in an extracted package directory, this approach may fail without further manual actions."
+	@echo "In such cases, Git will inform you about the file that couldn't be created as expected. Simply delete those files and relaunch the build."
+	@echo "This approach economizes time since most build artifacts do not need to be rebuilt, as the file dates should be the same as when you originally built them."
+	@echo "Only a minimal time is needed for rebuilding, which is also good for your SSD."
+	@echo "*** USE THIS APPROACH FIRST ***"
+	find ./build/ -type f -name ".canary" -print -delete
+	find ./install/*/* -print -exec rm -rf {} +
+
+real.gitclean_keep_packages_and_build:
+	@echo "Cleaning the repository using Git ignore file as a base..."
+	@echo "This will wipe everything not in the Git tree, but keep the 'packages' and 'build' directories."
+	git clean -fxd -e "packages" -e "build"

Makefile

@@ -28,8 +28,87 @@ Please refer to [Heads-wiki](https://osresearch.net) for your Heads' documentati
 
 Building heads
 ===
+
+Under QubesOS?
+====
+* Setup nix persistent layer under QubesOS (Thanks @rapenne-s !)
+  * https://dataswamp.org/~solene/2023-05-15-qubes-os-install-nix.html
+* Install docker under QubesOS (imperfect old article of mine. Better somewhere?)
+  * https://gist.github.com/tlaurion/9113983bbdead492735c8438cd14d6cd
+
+Build docker from nix develop layer locally
+====
+
+#### Set up Nix and flakes  
+
+* If you don't already have Nix, install it:  
+    * `[ -d /nix ] || sh <(curl -L https://nixos.org/nix/install) --no-daemon`  
+    * `. /home/user/.nix-profile/etc/profile.d/nix.sh`  
+* Enable flake support in nix  
+    * `mkdir -p ~/.config/nix`  
+    * `echo 'experimental-features = nix-command flakes' >>~/.config/nix/nix.conf`  
+
+#### Build image
+
+* Build nix developer local environment with flakes locked to specified versions  
+    * `nix --print-build-logs --verbose develop --ignore-environment --command true`  
+* Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image:  
+    * `nix build .#dockerImage && docker load < result` 
+
+Done!
+
+Your local docker image "linuxboot/heads:dev-env" is ready to use, reproducible for the specific Heads commit used and will produce ROMs reproducible for that Heads commit ID.
+
+Jump into nix develop created docker image for interactive workflow
+=====
+`docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env`
+
+
+From there you can use the docker image interactively.
+
 `make BOARD=board_name` where board_name is the name of the board directory under `./boards` directory.
 
+
+One such useful example is to build and test qemu board roms and test them through qemu/kvm/swtpm provided in the docker image. 
+Please refer to [qemu documentation](targets/qemu.md) for more information.
+
+Eg:
+```
+make BOARD=qemu-coreboot-fbwhiptail-tpm2 # Build rom, export public key to emulated usb storage from qemu runtime
+make BOARD=qemu-coreboot-fbwhiptail-tpm2 PUBKEY_ASC=~/pubkey.asc inject_gpg # Inject pubkey into rom image
+make BOARD=qemu-coreboot-fbwhiptail-tpm2 USB_TOKEN=Nitrokey3NFC PUBKEY_ASC=~/pubkey.asc ROOT_DISK_IMG=~/qemu-disks/debian-9.cow2 INSTALL_IMG=~/Downloads/debian-9.13.0-amd64-xfce-CD-1.iso run # Install
+```
+
+Alternatively, you can use locally built docker image to build a board ROM image in a single call.
+
+Eg:
+`docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=nitropad-nv41`
+
+
+Pull docker hub image to prepare reproducible ROMs as CircleCI in one call
+====
+```
+docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=x230-hotp-maximized
+docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=nitropad-nv41
+```
+
+Maintenance notes on docker image
+===
+Redo the steps above in case the flake.nix or nix.lock changes. Then publish on docker hub:
+
+```
+docker tag tlaurion/heads-dev-env:vx.y.z tlaurion/heads-dev-env:latest
+docker push tlaurion/heads-dev-env:latest
+```
+
+Notes:
+- Local builds can use ":latest" tag, which will use latest tested successful CircleCI run
+- To reproduce CirlceCI results, make sure to use the same versioned tag declared under .circleci/config.yml's "image:" 
+
+
+
+General notes on reproducible builds
+===
 In order to build reproducible firmware images, Heads builds a specific
 version of gcc and uses it to compile the Linux kernel and various tools
 that go into the initrd.  Unfortunately this means the first step is a

README.md

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # Clean all non-deterministric fields in a newc cpio file
 #
 # Items fixed:

bin/cpio-clean

@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+
+TEMPLATE="$1"
+RESULT="$2"
+BOARD_BUILD="$3"
+BRAND_NAME="$4"
+
+repo="$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")"
+# For both coreboot and Linux, the config file is in a board-
+# specific build directory, but the build occurs from the
+# parent of that directory.
+module_dir="$(realpath "$(dirname "$2")/..")"
+
+# Use relative paths since the config may be part of the ROM
+# artifacts, and relative paths won't depend on the workspace
+# absolute path.
+board_build_rel="$(realpath --relative-to "$module_dir" "$BOARD_BUILD")"
+repo_rel="$(realpath --relative-to "$module_dir" "$repo")"
+
+echo "board_build_rel=$board_build_rel"
+echo "repo_rel=$repo_rel"
+
+sed -e "s!@BOARD_BUILD_DIR@!${board_build_rel}!g" \
+    -e "s!@BLOB_DIR@!${repo_rel}/blobs!g" \
+    -e "s!@BRAND_DIR@!${repo_rel}/branding/$BRAND_NAME!g" \
+    -e "s!@BRAND_NAME@!$BRAND_NAME!g" \
+    "$TEMPLATE" > "$RESULT"

bin/prepare_module_config.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
 # Generate dm-verity hashes and sign the root hash
 #
 # Output looks like

bin/verity-sign

@@ -0,0 +1 @@
+mrc.bin

blobs/haswell/.gitignore

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

blobs/haswell/obtain-mrc

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # P7 ASUS
 
 function printusage {

blobs/p8z77-m_pro/download_BIOS_clean.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function printusage {
   echo "Usage: $0 -f <romdump> -m <me_cleaner>(optional) -i <ifdtool>(optional)"

blobs/t420/extract.sh

@@ -0,0 +1 @@
+me.bin

blobs/t440p/.gitignore

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

blobs/t440p/download-clean-me

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

blobs/t440p/extract

@@ -0,0 +1 @@
+me.bin

blobs/w541/.gitignore

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

blobs/w541/download-clean-me

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

blobs/w541/extract

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function printusage {
   echo "Usage: $0 -f <romdump> -m <me_cleaner>(optional) -i <ifdtool>(optional)"

blobs/x220/extract.sh

@@ -0,0 +1 @@
+me.bin

blobs/xx20/.gitignore

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 BLOBDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 

blobs/xx20/download_parse_me.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """ME7 Update binary parser."""
 

blobs/xx20/me7_update_parser.py

@@ -0,0 +1 @@
+me.bin

blobs/xx30/.gitignore

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function printusage {
   echo "Usage: $0 -m <me_cleaner>(optional)"

blobs/xx30/download_clean_me.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function printusage {
   echo "Usage: $0 -m <me_cleaner>(optional)"

blobs/xx30/download_clean_me_manually.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function printusage {
   echo "Usage: $0 -f <romdump> -m <me_cleaner>(optional) -i <ifdtool>(optional)"

blobs/xx30/extract.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # me_cleaner - Tool for partial deblobbing of Intel ME/TXE firmware images
 # Copyright (C) 2016-2018 Nicola Corna <[email protected]>

blobs/xx30/me_cleaner.py

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 BLOBDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 ROMPARSER="94a615302f89b94e70446270197e0f5138d678f3"

blobs/xx30/vbios_t530.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 BLOBDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 ROMPARSER="94a615302f89b94e70446270197e0f5138d678f3"

blobs/xx30/vbios_w530.sh

@@ -0,0 +1,2 @@
+ifd.bin
+me.bin