Merge pull request #447 from deniszh/DZ-Packaging

Fixing packaging

Author: deniszh

.github/workflows/codeql.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false

.github/workflows/golangci-lint.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   golangci:
     name: lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: golangci-lint

.github/workflows/release.yml

@@ -0,0 +1,93 @@
+name: Upload Packages to new release
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    env:
+      BINARY: ${{ github.event.repository.name }}
+      CGO_ENABLED: 0
+
+    outputs:
+      matrix: ${{ steps.build.outputs.matrix }}
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1
+
+      - uses: actions/checkout@v2
+        name: Checkout
+
+      - name: Test
+        run: make test
+        env:
+          CGO_ENABLED: 1
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
+      - name: Install packaging dependencies
+        run: |
+          gem install rake fpm:1.10.2 package_cloud
+          GO111MODULE=off go get github.com/mitchellh/gox
+
+      - name: Build packages
+        id: build
+        run: |
+          make gox-build fpm-deb fpm-rpm
+          make sum-files
+          ARTIFACTS=
+          # Upload all deb and rpm packages
+          for package in *deb *rpm; do ARTIFACTS=${ARTIFACTS}\"$package\",\ ; done
+          echo ::set-output name=matrix::{\"file\": [${ARTIFACTS} \"sha256sum\", \"md5sum\"]}
+
+      - name: Check version
+        id: check_version
+        run: |
+          ./out/${BINARY}-linux-amd64 -version
+          [ v$(./out/${BINARY}-linux-amd64 -version) = ${{ github.event.release.tag_name }} ]
+
+      - name: Artifact
+        id: artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: packages
+          retention-days: 1
+          path: |
+            *.deb
+            *.rpm
+            sha256sum
+            md5sum
+
+      - name: Push packages to the stable repo
+        run: make packagecloud-stable
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+
+  upload:
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJson(needs.build.outputs.matrix)}}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: packages
+      - name: Upload ${{ matrix.file }}
+        id: upload
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ${{ matrix.file }}
+          asset_name: ${{ matrix.file }}
+          asset_content_type: application/octet-stream

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
 
   tests:
     name: Test code
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         go:
@@ -28,20 +28,59 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
+      - name: Checkout to the latest tag
+        run: |
+          # Fetch all tags
+          git fetch --depth=1 --tags
+          # Get the latest tag
+          VERS=$(git tag -l | sort -Vr | head -n1)
+          # Fetch everything to the latest tag
+          git fetch --shallow-since=$(git log $VERS -1 --format=%at)
+        if: ${{ github.event_name == 'push' }} # only when built from master
+
+      - name: Build project
+        run: make
+
       - name: Test
+        run: make test
+        env:
+          CGO_ENABLED: 1
+
+      - name: Check Docker images
         run: |
-          make
-          make test
+          make image
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
+
       - name: Install packaging dependencies
         run: |
-          sudo apt-get install rpm ruby-dev
-          sudo gem install fpm -v 1.10.2
-      - name: Check packaging
-        run: |
+          gem install rake fpm:1.10.2 package_cloud
           GO111MODULE=off go get github.com/mitchellh/gox
-          make gox-build
-          make fpm-deb
-          make fpm-rpm
-      - name: Check Docker images
+
+      - name: Check packaging
         run: |
-          make image
+          make DEVEL=1 gox-build fpm-deb fpm-rpm
+          make sum-files
+
+      - name: Upload Artifact
+        if: ${{ matrix.go == '^1' }} # only upload artifact when built with latest go
+        id: artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: packages-${{ matrix.go }}
+          retention-days: 3
+          path: |
+            *.deb
+            *.rpm
+            sha256sum
+            md5sum
+
+      - name: Push packages to the autobuilds repo
+        if: ${{ github.event_name == 'push' && matrix.go == '^1' }} # only when built from master with latest go
+        run: make DEVEL=1 packagecloud-autobuilds
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+

.gitignore

@@ -4,4 +4,9 @@ _vendor/pkg
 .vscode
 _gopath
 .DS_Store
-.idea
+.idea
+out/
+md5sum
+sha256sum
+*.rpm
+*.deb

.travis.yml

@@ -3,9 +3,23 @@ MAINTAINER:="Roman Lomonosov <[email protected]>"
 DESCRIPTION:="Golang implementation of Graphite/Carbon server"
 MODULE:=github.com/go-graphite/go-carbon
 
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+
 GO ?= go
+export GOFLAGS +=  -mod=vendor
+export GO111MODULE := on
 TEMPDIR:=$(shell mktemp -d)
+
+DEVEL ?= 0
+ifeq ($(DEVEL), 0)
 VERSION:=$(shell sh -c 'grep "const Version" $(NAME).go  | cut -d\" -f2')
+else
+VERSION:=$(shell sh -c 'git describe --always --tags | sed -e "s/^v//i"')
+endif
+
+RPM_VERSION:=$(subst -,_,$(VERSION))
 BUILD ?= $(shell git describe --abbrev=4 --dirty --always --tags)
 
 all: $(NAME)
@@ -25,11 +39,11 @@ test:
 	make run-test COMMAND="test -race"
 
 gox-build:
-	rm -rf build
-	mkdir -p build/root/etc/$(NAME)/
-	cd build && $(GO) build $(MODULE)
-	gox -os="linux" -arch="amd64" -arch="386" -arch="arm64" -output="build/$(NAME)-{{.OS}}-{{.Arch}}" $(MODULE)
-	ls -la ./build/
+	rm -rf out
+	mkdir -p out
+	cd out && $(GO) build $(MODULE) && cd ..
+	gox -os="linux" -arch="amd64" -arch="386" -arch="arm64" -output="out/$(NAME)-{{.OS}}-{{.Arch}}" $(MODULE)
+	ls -la ./out/
 
 clean:
 	rm -f go-carbon build/* *deb *rpm
@@ -39,17 +53,17 @@ image:
 	docker build -t go-carbon .
 
 package-tree:
-	install -m 0755 -d build/root/lib/systemd/system
-	install -m 0755 -d build/root/etc/$(NAME)
-	install -m 0755 -d build/root/etc/logrotate.d
-	install -m 0755 -d build/root/etc/init.d
-	install -m 0644 deploy/$(NAME).service build/root/lib/systemd/system/$(NAME).service
-	./build/$(NAME)-linux-amd64 -config-print-default > deploy/$(NAME).conf
-	install -m 0644 deploy/$(NAME).conf build/root/etc/$(NAME)/$(NAME).conf
-	install -m 0644 deploy/storage-schemas.conf build/root/etc/$(NAME)/storage-schemas.conf
-	install -m 0644 deploy/storage-aggregation.conf build/root/etc/$(NAME)/storage-aggregation.conf
-	install -m 0644 deploy/$(NAME).logrotate build/root/etc/logrotate.d/$(NAME)
-	install -m 0755 deploy/$(NAME).init build/root/etc/init.d/$(NAME)
+	install -m 0755 -d out/root/lib/systemd/system
+	install -m 0755 -d out/root/etc/$(NAME)
+	install -m 0755 -d out/root/etc/logrotate.d
+	install -m 0755 -d out/root/etc/init.d
+	install -m 0755 -d out/root/etc/$(NAME)
+	./out/$(NAME)-linux-amd64 -config-print-default > out/root/etc/$(NAME)/$(NAME).conf
+	install -m 0644 deploy/$(NAME).service out/root/lib/systemd/system/$(NAME).service
+	install -m 0644 deploy/storage-schemas.conf out/root/etc/$(NAME)/storage-schemas.conf
+	install -m 0644 deploy/storage-aggregation.conf out/root/etc/$(NAME)/storage-aggregation.conf
+	install -m 0644 deploy/$(NAME).logrotate out/root/etc/logrotate.d/$(NAME)
+	install -m 0755 deploy/$(NAME).init out/root/etc/init.d/$(NAME)
 
 fpm-deb:
 	make fpm-build-deb ARCH=amd64
@@ -63,7 +77,7 @@ fpm-rpm:
 
 fpm-build-deb:
 	make package-tree
-	chmod 0755 build/$(NAME)-linux-$(ARCH)
+	chmod 0755 out/$(NAME)-linux-$(ARCH)
 	fpm -s dir -t deb -n $(NAME) -v $(VERSION) \
 		--deb-priority optional --category admin \
 		--package $(NAME)_$(VERSION)_$(ARCH).deb \
@@ -79,12 +93,12 @@ fpm-build-deb:
 		--after-install deploy/after_install.sh \
 		--after-upgrade deploy/after_install.sh \
 		--config-files /etc/ \
-		build/root/=/ \
-		build/$(NAME)-linux-$(ARCH)=/usr/bin/$(NAME)
+		out/root/=/ \
+		out/$(NAME)-linux-$(ARCH)=/usr/bin/$(NAME)
 
 fpm-build-rpm:
 	make package-tree
-	chmod 0755 build/$(NAME)-linux-$(ARCH)
+	chmod 0755 out/$(NAME)-linux-$(ARCH)
 	fpm -s dir -t rpm -n $(NAME) -v $(VERSION) \
 		--package $(NAME)-$(VERSION)-1.$(FILE_ARCH).rpm \
 		--force \
@@ -99,5 +113,39 @@ fpm-build-rpm:
 		--after-install deploy/after_install.sh \
 		--after-upgrade deploy/after_install.sh \
 		--config-files /etc/ \
-		build/root/=/ \
-		build/$(NAME)-linux-$(ARCH)=/usr/bin/$(NAME)
+		out/root/=/ \
+		out/$(NAME)-linux-$(ARCH)=/usr/bin/$(NAME)
+
+packagecloud-push-rpm: $(wildcard $(NAME)-$(RPM_VERSION)-1.*.rpm)
+	for pkg in $^; do
+		package_cloud push $(REPO)/el/7 $${pkg} || true
+		package_cloud push $(REPO)/el/8 $${pkg} || true
+	done
+
+packagecloud-push-deb: $(wildcard $(NAME)_$(VERSION)_*.deb)
+	for pkg in $^; do
+		package_cloud push $(REPO)/ubuntu/xenial   $${pkg} || true
+		package_cloud push $(REPO)/ubuntu/bionic   $${pkg} || true
+		package_cloud push $(REPO)/ubuntu/focal    $${pkg} || true
+		package_cloud push $(REPO)/debian/stretch  $${pkg} || true
+		package_cloud push $(REPO)/debian/buster   $${pkg} || true
+		package_cloud push $(REPO)/debian/bullseye $${pkg} || true
+	done
+
+packagecloud-push:
+	@$(MAKE) packagecloud-push-rpm
+	@$(MAKE) packagecloud-push-deb
+
+packagecloud-autobuilds:
+	$(MAKE) packagecloud-push REPO=go-graphite/autobuilds
+
+packagecloud-stable:
+	$(MAKE) packagecloud-push REPO=go-graphite/stable
+
+sum-files: | sha256sum md5sum
+
+md5sum:
+	md5sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > md5sum
+
+sha256sum:
+	sha256sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > sha256sum

Makefile

@@ -72,6 +72,18 @@ cd go-carbon
 make
 ```
 
+We are using <a href="https://packagecloud.io/"><img alt="Private Maven, RPM, DEB, PyPi and RubyGem Repository | packagecloud" height="46" src="https://packagecloud.io/images/packagecloud-badge.png" width="158" /></a> to host our packages!
+
+At this moment we are building deb and rpm packages for i386, amd64 and arm64 architectures. Installation guides are available on packagecloud (see the links below).
+
+Stable versions: [Stable repo](https://packagecloud.io/go-graphite/stable/install)
+
+Autobuilds (master, might be unstable): [Autobuild repo](https://packagecloud.io/go-graphite/autobuilds/install)
+
+We're uploading Docker images to [Docker Hub](https://hub.docker.com/repository/docker/gographite/go-carbon/general) and [ghcr.io](https://github.com/go-graphite/go-carbon/pkgs/container/go-carbon)
+
+Also, you can download test packages from build artifacts: Go to list of [test runs](https://github.com/go-graphite/go-carbon/actions/workflows/tests.yml), click on PR name, and click on *"packages-^1"* under *"Artifact"* section.
+
 ## Configuration
 
 ```