Merge branch 'feature/android-11-and-debian-10-base-image'

Author: ypid

.gitignore

@@ -1,6 +1,13 @@
-build/*
-config/keys/*
+build
 config/env/*
-config/container/Dockerfile.*
+config/container/Dockerfile-*
+config/manifests/**/*.yml
 release/*
+
+# Old path, please manually move to new path.
+config/keys
+
+# New keys path.
+keys
+
 .*

Makefile

@@ -1,17 +1,21 @@
+SHELL = /bin/bash -o nounset -o pipefail -o errexit
+MAKEFLAGS += --no-builtin-rules
+.SUFFIXES:
+
 ## Argument Variables ##
 
-CPUS := $(shell nproc)
-MEMORY := 10000
-DISK := 300000
-DEVICE :=
-BACKEND := local
-CHANNEL := beta
-BUILD := user
-FLAVOR := aosp
-IMAGE := hashbang/aosp-build:latest
-IMAGE_OPTIONS :=
-NAME := aosp-build-$(FLAVOR)-$(BACKEND)
-SHELL := /bin/bash
+CPUS = $(shell nproc)
+MEMORY = 10000
+DISK = 300000
+DEVICE =
+BACKEND = local
+CHANNEL = beta
+FLAVOR = aosp
+IMAGE = hashbang/aosp-build:latest
+IMAGE_OPTIONS =
+RUN_OPTIONS =
+NAME = aosp-build-$(FLAVOR)-$(BACKEND)
+REQUIRED_FREE_SPACE_IN_GIB = 120
 
 -include $(PWD)/config/env/$(BACKEND).env
 
@@ -25,20 +29,24 @@ default: machine image fetch tools keys build release
 ## Primary Targets ##
 
 .PHONY: fetch
-fetch: submodule-update machine image
+fetch:
 	$(contain) fetch
 
 .PHONY: keys
 keys:
-	$(contain) keys
+	$(contain-keys) keys
+
+.PHONY: review
+review:
+	$(contain) review
 
 .PHONY: build
-build:
+build: ensure-enough-free-disk-space
 	$(contain) build
 
 .PHONY: release
 release:
-	$(contain) release
+	$(contain-keys) release
 
 .PHONY: publish
 publish:
@@ -52,28 +60,41 @@ clean:
 mrproper: storage-delete machine-delete
 	rm -rf build
 
-
 ## Secondary Targets ##
 
+config/container/Dockerfile: config/container/Dockerfile.j2 config/container/render_template
+	./config/container/render_template "$<" "{\"tags\":[]}" > "$@"
+
+## Support for different Docker image variants.
+config/container/Dockerfile-golang:
+config/container/Dockerfile-latest:
+config/container/Dockerfile-%: config/container/Dockerfile.j2 config/container/render_template
+	./config/container/render_template "$<" "{\"tags\":[\"$*\"]}" > "$@"
+
 .PHONY: image
-image:
+image: config/container/Dockerfile
 	$(docker) build \
 		--tag $(IMAGE) \
-		--file $(PWD)/config/container/Dockerfile \
+		--file "$(PWD)/$<" \
 		$(IMAGE_OPTIONS) \
 		$(PWD)
 
-config/container/Dockerfile.minimal: config/container/Dockerfile config/container/render_template
-	./config/container/render_template "$<" | grep -v '^#\s*$$' > "$@"
-
-.PHONY: image-minimal
-image-minimal: config/container/Dockerfile.minimal
+.PHONY: image-%
+image-golang:
+image-latest:
+image-%: config/container/Dockerfile-%
 	$(docker) build \
 		--tag $(IMAGE) \
 		--file "$(PWD)/$<" \
 		$(IMAGE_OPTIONS) \
 		$(PWD)
 
+## Note that the `image-latest` target should be used for pinning.
+.PHONY: config/container/packages-pinned.list
+config/container/packages-pinned.list:
+	$(contain-no-tty) pin-packages > "$@"
+
+
 .PHONY: tools
 tools:
 	mkdir -p config/keys build/base release build/external
@@ -97,14 +118,14 @@ kernel:
 .PHONY: latest
 latest: config submodule-latest fetch
 
-.PHONY: manifest
-manifest: config
-	$(contain) bash -c "source <(environment) && manifest"
-
 .PHONY: config
 config:
 	$(contain) bash -c "source <(environment) && config"
 
+.PHONY: manifest
+manifest:
+	$(contain) bash -c "source <(environment) && manifest"
+
 .PHONY: test-repro
 test-repro:
 	$(contain) test-repro
@@ -114,18 +135,16 @@ test: test-repro
 
 .PHONY: patches
 patches:
-	@$(contain) bash -c "cd base; repo diff --absolute"
+	@$(contain) bash -c "cd build/base && repo diff --absolute"
 
 .PHONY: shell
 shell:
-	$(docker) inspect "$(NAME)" \
-	&& $(docker) exec --interactive --tty "$(NAME)" shell \
-	|| $(contain) shell
+	$(docker) exec --interactive --tty "$(NAME)" shell \
+		|| $(contain) shell
 
 .PHONY: monitor
 monitor:
-	$(docker) inspect "$(NAME)" \
-	&& $(docker) exec --interactive --tty "$(NAME)" htop
+	$(docker) exec --interactive --tty "$(NAME)" htop
 
 .PHONY: install
 install: tools
@@ -250,22 +269,63 @@ endif
 userid = $(shell id -u)
 groupid = $(shell id -g)
 docker_machine = docker-machine --storage-path "${PWD}/build/machine"
-contain := \
+
+# Can be used mount aosp-build directory to /opt/aosp-build to allow fast
+# development without the need to rebuild the container image all the time.
+# See HashbangMobile for example.
+contain-base-extend =
+
+contain-base = \
 	$(docker) run \
 		--rm \
-		--tty \
 		--interactive \
 		--name "$(NAME)" \
 		--hostname "$(NAME)" \
 		--user $(userid):$(groupid) \
 		--env DEVICE=$(DEVICE) \
+		--privileged \
 		--security-opt seccomp=unconfined \
 		--volume $(PWD)/config:/home/build/config \
 		--volume $(PWD)/release:/home/build/release \
 		--volume $(PWD)/scripts:/home/build/scripts \
-		$(storage_flags) \
+		$(contain-base-extend) \
+		$(RUN_OPTIONS) \
+		--shm-size="1g" \
+		$(storage_flags)
+
+contain-no-tty = \
+	$(contain-base) \
+		$(IMAGE)
+
+contain-keys = \
+	$(contain-base) \
+		--tty \
+		--volume $(PWD)/keys:/home/build/keys \
+		$(IMAGE)
+
+contain = \
+	$(contain-base) \
+		--tty \
 		$(IMAGE)
 
+## Helpers ##
+
+ensure-git-status-clean:
+	@if [ -z "$(shell git status --porcelain=v2)" ]; then \
+		echo "git status has no output. Working tree is clean."; \
+	else \
+		git status; \
+		echo "Working tree is not clean as required. Exiting."; \
+		exit 1; \
+	fi
+
+ensure-enough-free-disk-space:
+	@free_space=$(shell df -k --output=avail "$$PWD" | tail -n1); \
+	needed_free_space=$$(( $(REQUIRED_FREE_SPACE_IN_GIB) * 1024 * 1024 )); \
+	if [[ $$free_space -lt $$needed_free_space ]]; then \
+		echo "Not enought free space. $(REQUIRED_FREE_SPACE_IN_GIB) GiB are required." 1>&2; \
+		exit 1; \
+	fi
 
 ## Required Binary Check ##
 

README.md

@@ -15,7 +15,7 @@ of the Makefile and config.yml from this repo, along with any desired patches.
 
 ## Support ##
 
-Please join us on IRC: ircs://irc.hashbang.sh/#!os
+Please join us on IRC: ircs://irc.hashbang.sh/#!mobile
 
 ## Features ##
 
@@ -42,72 +42,12 @@ Please join us on IRC: ircs://irc.hashbang.sh/#!os
 
 ## Install ##
 
-### Requirements ###
+Refer to [GrapheneOS CLI install].
 
- * [Android Developer Tools][4]
+[GrapheneOS CLI install]: https://grapheneos.org/install/cli
 
-[4]: https://developer.android.com/studio/releases/platform-tools
+### Notes
 
-### Connect
-
- 1. Go to "Settings > About Phone"
- 2. Tap "Build number" 7 times.
- 3. Go to "Settings > System > Advanced > Developer options"
- 4. Enable "USB Debugging"
- 5. Connect to device to laptop via short USB C cable
- 6. Hit "OK" on "Allow USB Debugging?" prompt on device if present.
- 7. Verify ADB connectivity
-   ```
-   adb devices
-   ```
-   Note: Should return something like: "7CKY1QD3F       device"
-
-### Flash
-
- 1. Extract
-
-   ```
-   unzip crosshatch-PQ1A.181205.006-factory-1947dcec.zip
-   cd crosshatch-PQ1A.181205.006
-   ```
-
- 2. [Connect](#Connect)
- 3. Go to "Settings > System > Advanced > Developer options"
- 4. Enable "OEM Unlocking"
- 5. Unlock the bootloader via ADB
-
-   ```
-   adb reboot bootloader
-   fastboot flashing unlock
-   ```
-   Note: You must manually accept prompt on device.
-
- 6. Flash new factory images
-
-   ```
-   ./flash-all.sh
-  ```
-
-### Harden
-
- 1. [Connect](#Connect)
- 2. Lock the bootloader
-   ```
-   adb reboot bootloader
-   fastboot flashing lock
-   ```
- 3. Go to "Settings > About Phone"
- 4. Tap "Build number" 7 times.
- 5. Go to "Settings > System > Advanced > Developer options"
- 6. Disable "OEM unlocking"
- 7. Reboot
- 8. Verify boot message: "Your device is loading a different operating system"
- 9. Go to "Settings > System > Advanced > Developer options"
- 10. Verify "OEM unlocking" is still disabled
-
-#### Notes
-
-  * Failure to run these hardening steps means -anyone- can flash your device.
   * Past this point if signing keys are lost, all devices are bricked. Backup!
 
 ### Update ###
@@ -126,11 +66,27 @@ Please join us on IRC: ircs://irc.hashbang.sh/#!os
 
 ## Build ##
 
+Most of the dependencies are "contained". Only minimal software requirements
+exist for the controlling host that cannot be contained easily because of the
+bootstrapping problem:
+
+* GNU core utilities
+* GNU Make
+* Python 3 dependencies: jinja2
+
+They should be packaged by your distribution under the following names (adjust
+slight distro differences yourself):
+
+```
+coreutils make python3 python3-jinja2
+```
+
 ### Backends ###
 
 #### Local
 
 ##### Requirements
+
  * Docker 10+
  * x86_64 CPU
  * 10GB+ available memory
@@ -250,13 +206,41 @@ make diff > patches/my-feature.patch
 make install
 ```
 
-#### Update ####
+### Release ###
 
-Build latest config from upstream sources:
+1. Update references to latest upstream sources.
 
-```
-make DEVICE=crosshatch manifest
-```
+  ```
+  make config
+  ```
+
+1. Regenerate the git-repo XML manifest files.
+
+  ```
+  make manifest
+  ```
+
+1. Build all targets impacted by given change
+
+  ```
+  make DEVICE=crosshatch release
+  ```
+
+1. Commit changes to a PR
+
+## Review ##
+
+Patchsets that base on AOSP will carry their patchset forward using `git
+rebase`. In case you use aosp-build you might be interested in an ongoing
+review of this patchset across rebases. For this, checkout `make review`.
+
+Refer to https://github.com/ypid/android-review for one public instance of such
+a review.
+
+### How it works? ###
+
+We use the hash locked manifest that [aosp-build] produces from AOSP to
+whatever you have checked out.
 
 ## Notes ##