Merge pull request #33 from KiFoundation/improve_makefile

Improve makefile

Author: WinterNis

e2e.Dockerfile renamed to Dockerfile

@@ -1,30 +1,35 @@
 ARG IMG_TAG=latest
+ARG PLATFORM="linux/amd64"
 
 # Compile the kid binary
-FROM golang:1.17-alpine3.15 AS kid-builder
+FROM --platform=${PLATFORM} golang:1.17.10-alpine3.15 AS kid-builder
 WORKDIR /src/app/
 COPY go.mod go.sum* ./
 RUN go mod download
 COPY . .
 
 # From https://github.com/CosmWasm/wasmd/blob/master/Dockerfile
 # For more details see https://github.com/CosmWasm/wasmvm#builds-of-libwasmvm
-ARG arch=x86_64
+ARG ARCH=x86_64
 ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0-beta10/libwasmvm_muslc.aarch64.a /lib/libwasmvm_muslc.aarch64.a
 ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0-beta10/libwasmvm_muslc.x86_64.a /lib/libwasmvm_muslc.x86_64.a
 RUN sha256sum /lib/libwasmvm_muslc.aarch64.a | grep 5b7abfdd307568f5339e2bea1523a6aa767cf57d6a8c72bc813476d790918e44
 RUN sha256sum /lib/libwasmvm_muslc.x86_64.a | grep 2f44efa9c6c1cda138bd1f46d8d53c5ebfe1f4a53cf3457b01db86472c4917ac
-RUN cp /lib/libwasmvm_muslc.${arch}.a /lib/libwasmvm_muslc.a
+RUN cp /lib/libwasmvm_muslc.${ARCH}.a /lib/libwasmvm_muslc.a
 
 ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev python3
 RUN apk add --no-cache $PACKAGES
 RUN set -eux; apk add --no-cache ca-certificates build-base;
-RUN BUILD_TAGS=muslc LINK_STATICALLY=true make install
+
+ARG BUILD_TARGET_PREFIX=""
+ARG VERSION=""
+RUN BUILD_TAGS=muslc LINK_STATICALLY=true LDFLAGS=-buildid=$VERSION make build$BUILD_TARGET_PREFIX
 
 # Add to a distroless container
-FROM gcr.io/distroless/cc:$IMG_TAG
+ARG PLATFORM="linux/amd64"
+FROM --platform=${PLATFORM} gcr.io/distroless/cc:$IMG_TAG
 ARG IMG_TAG
-COPY --from=kid-builder /go/bin/kid /usr/local/bin/
+COPY --from=kid-builder /src/app/build/kid /usr/local/bin/
 EXPOSE 26656 26657 1317 9090
 
 ENTRYPOINT ["kid", "start"]

Makefile

@@ -109,16 +109,25 @@ $(BUILD_TARGETS): go.sum $(BUILDDIR)/
 $(BUILDDIR)/:
 	mkdir -p $(BUILDDIR)/
 
-build-reproducible: go.sum
-	$(DOCKER) rm latest-build || true
-	$(DOCKER) run --volume=$(CURDIR):/sources:ro \
-        --env TARGET_PLATFORMS='linux/amd64 darwin/amd64 linux/arm64 windows/amd64' \
-        --env APP=kid \
-        --env VERSION=$(VERSION) \
-        --env COMMIT=$(COMMIT) \
-        --env LEDGER_ENABLED=$(LEDGER_ENABLED) \
-        --name latest-build cosmossdk/rbuilder:latest
-	$(DOCKER) cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/
+build-reproducible-all: build-reproducible-amd64 build-reproducible-arm64
+
+build-reproducible-amd64:
+	ARCH=x86_64 PLATFORM=linux/amd64 $(MAKE) build-reproducible-generic
+
+build-reproducible-arm64:
+	ARCH=aarch64 PLATFORM=linux/arm64 $(MAKE) build-reproducible-generic
+	
+build-reproducible-generic: go.sum
+	$(DOCKER) rm $(subst /,-,latest-build-$(PLATFORM)) || true
+	DOCKER_BUILDKIT=1 $(DOCKER) build -t latest-build-$(PLATFORM) \
+		--build-arg ARCH=$(ARCH) \
+		--build-arg PLATFORM=$(PLATFORM) \
+		--build-arg BUILD_TARGET_PREFIX="$(BUILD_TARGET_PREFIX)" \
+		--build-arg VERSION="$(VERSION)" \
+		-f Dockerfile .
+	$(DOCKER) create -ti --name $(subst /,-,latest-build-$(PLATFORM)) latest-build-$(PLATFORM) kid
+	mkdir -p $(CURDIR)/build/$(NETWORK)/$(PLATFORM)/
+	$(DOCKER) cp -a $(subst /,-,latest-build-$(PLATFORM)):/usr/local/bin/kid $(CURDIR)/build/$(NETWORK)/$(PLATFORM)/kid
 
 build-linux: go.sum
 	LEDGER_ENABLED=false GOOS=linux GOARCH=amd64 $(MAKE) build
@@ -150,10 +159,17 @@ distclean: clean
 ###                                 Testnet                                 ###
 ###############################################################################
 
+build-testnet-reproducible-all: build-testnet-reproducible-amd64 build-testnet-reproducible-arm64
+
+build-testnet-reproducible-amd64:
+	ARCH=x86_64 PLATFORM=linux/amd64 NETWORK=Testnet ADDRESS_PREFIX=tki BUILD_TARGET_PREFIX="-testnet" $(MAKE) build-reproducible-generic
+
+build-testnet-reproducible-arm64:
+	ARCH=aarch64 PLATFORM=linux/arm64 NETWORK=Testnet ADDRESS_PREFIX=tki BUILD_TARGET_PREFIX="-testnet" $(MAKE) build-reproducible-generic
+
 build-testnet: go.sum
 	NETWORK=Testnet ADDRESS_PREFIX=tki $(MAKE) build
 
-
 ###############################################################################
 ###                                 Devdoc                                  ###
 ###############################################################################
@@ -203,7 +219,7 @@ benchmark:
 	@go test -mod=readonly -bench=. ./...
 
 docker-build-debug:
-	@docker build -t cosmos/kid-e2e --build-arg IMG_TAG=debug -f e2e.Dockerfile .
+	@docker build -t cosmos/kid-e2e --build-arg IMG_TAG=debug -f Dockerfile .
 
 ###############################################################################
 ###                                Linting                                  ###

README.md

@@ -3,22 +3,23 @@
 </p>
 
 # Ki Tools
-This repository hosts `ki-tools`, a set of tools that allow to deploy and run Kichain nodes.
 
+This repository hosts `kid`, the implementation of the kichain protocol, based on Cosmos-SDK.
 
 ## Quick Start
 
-### Install Golang
+### Install Golang (linux)
+
 To install Go, visit the Go download page and copy the link of the latest Go release for Linux systems, download and unzip the archive file as follows:
 
-```
-wget https://dl.google.com/go/go1.16.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.16.linux-amd64.tar.gz
+```bash
+wget https://dl.google.com/go/go1.17.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
 ```
 
 Finally, export the Go paths like so:
 
-```
+```bash
 mkdir -p $HOME/go/bin
 PATH=$PATH:/usr/local/go/bin
 echo "export PATH=$PATH:$(go env GOPATH)/bin" >> ~/.bash_profile
@@ -27,46 +28,141 @@ source ~/.bash_profile
 
 To test the Go installation,  use the `version` command to check the downloaded version as follows :
 
-```
+```bash
 go version
 ```
 
 This should output :
 
-```
-go version go1.16 linux/amd64
+```bash
+go version go1.17 linux/amd64
 ```
 
-### Install ki-tools
-Start by clone this repository. If you are here, you most likely have git installed already, otherwise just run:
+### Build kid (linux)
 
-```
+Start by cloning this repository. If you are here, you most likely have git installed already, otherwise just run:
+
+```bash
 sudo apt install git
 ```
 
 And now you can clone the repository as follows:
-```
+
+```bash
 git clone https://github.com/KiFoundation/ki-tools.git
 ```
 
 If your are starting with a clean ubuntu install you might need to install the `build-essential` package:
 
-```
+```bash
 sudo apt update
 sudo apt install build-essential
 ```
 
 Finally, navigate to the repository folder and install the tools as follows:
 
-```
+```bash
 cd ki-tools
 make install
 ```
 
-To test the installation, check the downloaded version as follows :
-```
+To test the installation, check the downloaded version as follows:
+
+```bash
 kid version --long
 ```
 
+### Building for testnet
+
+Testnet token name (tki) is different from mainnet (xki). Testnet binaries need to support this difference.
+To build a testnet binary, run as follows:
+
+```bash
+cd ki-tools
+make build-testnet
+```
+
+You can also build a static testnet binary as explained in the following section.
+
+### Build a static kid binary
+
+#### **Why building static binaries ?**
+
+Static binaries hermetically contain libraries that they are using. Dynamic binaries rely on libraries located elsewhere on the system, the binary only containing the address of the library.
+There are multiple pro and cons of **static vs dynamic**.
+
+For `kid`, we provide a set a tool to build static binaries, as we want to ensure that:
+
+* Binaries and dependencies are consistent accross the validators set.
+* Build result is reproducible and can be verified by every user.
+
+Using static, verified, binaries ensures that all nodes mainteners are running the same version of dependencies, the `cosmwasm` particulary.
+You can see this as a protection against unexpected consensus failures due to binary dependency mismatch.
+
+Static binaries integrity can be checked by comparing sha256sum.
+
+#### **Requirements**
+
+To build a static binary, we rely on `Docker` and `alpine` docker images.
+
+##### **Install docker**
+
+To install docker, follow [the official procedure for your platform](https://docs.docker.com/get-docker/)
+
+##### **Install Qemu (Optional)**
+
+If you want to build `kid` to run on a platform different from your build platform, you will need to install `Qemu`:
+
+For linux users:
+
+```bash
+sudo apt-get install qemu binfmt-support qemu-user-static 
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
+For macOS users, `Qemu` should already be installed and configured when installing `Docker Desktop`.
+
+##### **Build**
+
+```bash
+cd ki-tools
+```
+
+Build an `amd64` static binary
+
+```bash
+make build-reproducible-amd64           # For mainnet
+make build-reproducible-testnet-amd64   # For testnet
+```
+
+Build an `arm64` static binary
+
+```bash
+make build-reproducible-arm64           # For mainnet
+make build-reproducible-testnet-arm64   # For testnet
+```
+
+**Note:** arm64 compatiblity is proposed but not guaranteed. We recommend users to run on amd64 platforms.
+
+Built binaries can then be found in the **build** folder
+
+```bash
+$ tree build
+build
+├── Mainnet
+│   └── linux
+│       ├── amd64
+│       │   └── kid
+│       └── arm64
+│           └── kid
+└── Testnet
+    └── linux
+        ├── amd64
+        │   └── kid
+        └── arm64
+            └── kid
+```
+
 ## Disclaimer
-The `ki-tools` is a modified clone of the `gaia` project. More about the latter can be found [here](https://github.com/cosmos/gaia).
+
+The `ki-tools` is a modified clone of the `gaia` project. More about the latter can be found [here](https://github.com/cosmos/gaia).