feat: improve and simplify targets.json (#125)

Co-authored-by: raphaelcoeffic <[email protected]>
Co-authored-by: Peter Feerick <[email protected]>

Author: raphaelcoeffic

Makefile

@@ -7,12 +7,10 @@ migrate:
 	go run cmd/db/main.go -migrate
 
 lint:
-	go install github.com/golangci/golangci-lint/cmd/[email protected]
 	golangci-lint run ./...
 
 lint-fix:
 	go mod tidy
-	go install github.com/golangci/golangci-lint/cmd/[email protected]
 	golangci-lint run --fix ./...
 
 build:

README.md

@@ -32,13 +32,13 @@ Linux operating system with:
 
 First clone this repository:
 
-``` shell
+```shell
 git clone https://github.com/EdgeTX/cloudbuild.git
 ```
 
 Then you will need to configure some things:
 
-``` shell
+```shell
 # Create config from template
 cp api.env.example api.env
 ```
@@ -47,7 +47,7 @@ For a quick test, all you need to edit is external URL (as it will be seen by we
 and the directory in which the firmwares are stored (which should be shared among all 
 containers as a volume).
 
-```
+```env
 EBUILD_STORAGE_PATH=/home/rootless/src/static/firmwares
 EBUILD_DOWNLOAD_URL=http://localhost:3000/firmwares
 ```
@@ -56,7 +56,7 @@ EBUILD_DOWNLOAD_URL=http://localhost:3000/firmwares
 
 The runtime container image can be built with:
 
-``` shell
+```shell
 docker compose build api
 ```
 
@@ -66,19 +66,19 @@ docker compose build api
 
 Start the database first:
 
-``` shell
+```shell
 docker compose up -d db
 ```
 
 Then start the API:
 
-``` shell
+```shell
 docker compose up -d api
 ```
 
 And setup the database schema with:
 
-``` shell
+```shell
 docker exec -it cloudbuild-api-1 ./ebuild db migrate
 ```
 
@@ -88,7 +88,7 @@ Once the database is setup, proceed to the next step to start the workers as wel
 
 Then the rest of the stack can be run all together:
 
-``` shell
+```shell
 docker compose up -d --scale worker=2
 ```
 
@@ -100,7 +100,7 @@ You can increase the number of workers if you want to build more firmwares in pa
 To offload serving the firmware files to a S3 compatible storage, a few configuration
 parameters need to be set additionally in the environment file (`api.env`):
 
-```
+```env
 # Common settings for all provider
 EBUILD_STORAGE_TYPE: S3
 EBUILD_S3_ACCESS_KEY: myAccessKey
@@ -118,15 +118,49 @@ EBUILD_DOWNLOAD_URL: https://bucket.s3.super-provider.com
 
 To be able to use the administrative UI, a token must be generated for every user:
 
-``` shell
+```shell
 docker exec -it cloudbuild-api-1 ./ebuild auth create [some name]
 AccessKey: [some access key]
 SecretKey: [very secret key]
 ```
 
 The token can be later removed:
 
-``` shell
+```shell
 docker exec -it cloudbuild-api-1 ./ebuild auth remove [Access Key]
 token [Access Key] removed
 ```
+
+## Using a local Git mirror
+
+To speed up builds, it is possible to use a local mirror by changing
+the repository URL:
+```env
+EBUILD_SRC_REPO=http://local-mirror:8080/EdgeTX/edgetx
+```
+
+[cloudbuild-mirror](https://github.com/EdgeTX/cloudbuild-mirror) can be
+used as a local mirror. It also supports automatic refreshing of the
+targets (see next section).
+
+With this configuration only, the submodules will still be fetched from `http://github.com`,
+as the full URL is encoded into the respective `.gitmodules` files. Fortunately, Git offers
+a simple to override this with [insteadOf](https://git-scm.com/docs/git-config#Documentation/git-config.txt-urlbaseinsteadOf).
+
+Just mount this file into the workers at `/etc/gitconfig`:
+```
+[url "http://local-mirror:8080"]
+  insteadOf = https://github.com
+```
+
+## Refresh targets automatically from URL
+
+It is also possible to fetch `targets.json` from a URL instead of a local
+file. In that case, the targets are refreshed automatically (defaults to 5 minutes interval).
+
+```env
+# Use cloudbuild mirrored repository as the source for targets.json
+EBUILD_TARGETS=http://local-mirror:8080/gitweb.cgi?p=EdgeTX/cloudbuild.git;a=blob_plain;f=targets.json;hb=HEAD
+# This is the default refresh interval in seconds
+EBUILD_TARGETS_REFRESH_INTERVAL=300
+```

artifactory/artifactory_test.go

@@ -28,10 +28,11 @@ import (
 func init() {
 	log.SetOutput(os.Stdout)
 	log.SetLevel(log.DebugLevel)
-	err := targets.ReadTargetsDefFromBytes([]byte(targetsJSON))
+	defs, err := targets.ReadTargetsDefFromBytes([]byte(targetsJSON), "")
 	if err != nil {
 		panic(err)
 	}
+	targets.SetTargets(defs)
 	request = artifactory.NewBuildRequestWithParams(commitRef, target, flags)
 }
 

cmd/ebuild/run/cmd.go

@@ -40,11 +40,19 @@ func (s *serverRunner) runAPI(cmd *cobra.Command, args []string) {
 		fmt.Printf("failed to migrate database: %s", err)
 		os.Exit(1)
 	}
-	if err := targets.ReadTargetsDef("./targets.json"); err != nil {
+	if defs, err := targets.ReadTargetsDef(
+		s.opts.TargetsDef,
+		s.opts.SourceRepository,
+	); err != nil {
 		fmt.Printf("failed to read targets: %s", err)
 		os.Exit(1)
+	} else {
+		targets.SetTargets(defs)
 	}
-	go targets.Updater(time.Minute * 5)
+	go targets.Updater(
+		time.Second*time.Duration(s.opts.TargetsRefreshInterval),
+		s.opts.SourceRepository,
+	)
 	art, err := artifactory.NewFromConfig(s.ctx, s.opts)
 	if err != nil {
 		fmt.Printf("failed to create artifactory: %s", err)
@@ -131,7 +139,6 @@ func NewAPICommand(s *serverRunner) *cobra.Command {
 
 func NewWorkerCommand(s *serverRunner) *cobra.Command {
 	cmd := s.makeCmd("worker", "Run a cloudbuild worker", s.runWorker)
-	s.opts.BindWorkerOpts(cmd)
 	return cmd
 }
 

config/options.go

@@ -50,8 +50,8 @@ func LogLevelDecodeHookFunc() mapstructure.DecodeHookFunc {
 	return func(
 		f reflect.Type,
 		t reflect.Type,
-		data interface{},
-	) (interface{}, error) {
+		data any,
+	) (any, error) {
 		if f.Kind() != reflect.String {
 			return data, nil
 		}
@@ -85,6 +85,10 @@ type CloudbuildOpts struct {
 	DatabaseDSN  string `mapstructure:"database-dsn"`
 	DatabaseHost string `mapstructure:"database-host"`
 
+	// Targets options:
+	TargetsDef             string `mapstructure:"targets"`
+	TargetsRefreshInterval uint32 `mapstructure:"targets-refresh-interval"`
+
 	// Build options:
 	BuildImage       string `mapstructure:"build-img"`
 	SourceRepository string `mapstructure:"src-repo"`
@@ -104,14 +108,16 @@ type CloudbuildOpts struct {
 
 func NewOpts(v *viper.Viper) *CloudbuildOpts {
 	return &CloudbuildOpts{
-		Viper:            v,
-		LogLevel:         InfoLevel,
-		HTTPBindPort:     3000,
-		BuildImage:       "ghcr.io/edgetx/edgetx-builder",
-		SourceRepository: "https://github.com/EdgeTX/edgetx.git",
-		DownloadURL:      "http://localhost:3000",
-		StorageType:      "FILE_SYSTEM_STORAGE",
-		StoragePath:      "/tmp",
+		Viper:                  v,
+		LogLevel:               InfoLevel,
+		HTTPBindPort:           3000,
+		TargetsDef:             "file:///targets.json",
+		TargetsRefreshInterval: 300,
+		BuildImage:             "ghcr.io/edgetx/edgetx-builder",
+		SourceRepository:       "https://github.com/EdgeTX/edgetx.git",
+		DownloadURL:            "http://localhost:3000",
+		StorageType:            "FILE_SYSTEM_STORAGE",
+		StoragePath:            "/tmp",
 	}
 }
 
@@ -162,18 +168,28 @@ func (o *CloudbuildOpts) BindBuildOpts(c *cobra.Command) {
 	c.PersistentFlags().StringVar(
 		&o.BuildImage, "build-img", o.BuildImage, "Build docker image",
 	)
+	c.Flags().StringVar(
+		&o.SourceRepository, "src-repo", o.SourceRepository, "Source repository",
+	)
 }
 
 func (o *CloudbuildOpts) BindAPIOpts(c *cobra.Command) {
-	c.Flags().Uint16VarP(&o.HTTPBindPort, "port", "p", o.HTTPBindPort, "HTTP listen port")
-	c.Flags().IPVarP(&o.HTTPBindAddress, "listen-ip", "l", net.IPv4zero, "HTTP listen IP")
-	c.Flags().StringVarP(
-		&o.DownloadURL, "download-url", "u", o.DownloadURL, "Artifact download URL")
-}
-
-func (o *CloudbuildOpts) BindWorkerOpts(c *cobra.Command) {
+	c.Flags().Uint16VarP(
+		&o.HTTPBindPort, "port", "p", o.HTTPBindPort, "HTTP listen port",
+	)
+	c.Flags().IPVarP(
+		&o.HTTPBindAddress, "listen-ip", "l", net.IPv4zero, "HTTP listen IP",
+	)
 	c.Flags().StringVar(
-		&o.SourceRepository, "src-repo", o.SourceRepository, "Source repository")
+		&o.TargetsDef, "targets", o.TargetsDef, "Targets definition",
+	)
+	c.Flags().Uint32Var(
+		&o.TargetsRefreshInterval, "targets-refresh-interval",
+		o.TargetsRefreshInterval, "Targets refresh interval",
+	)
+	c.Flags().StringVarP(
+		&o.DownloadURL, "download-url", "u", o.DownloadURL, "Artifact download URL",
+	)
 }
 
 func (o *CloudbuildOpts) Unmarshal() error {

go.mod

@@ -3,30 +3,35 @@ module github.com/edgetx/cloudbuild
 go 1.24.1
 
 require (
+	github.com/Masterminds/semver/v3 v3.4.0
 	github.com/aws/aws-sdk-go-v2 v1.18.0
 	github.com/aws/aws-sdk-go-v2/config v1.18.23
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.22
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.9.1
+	github.com/go-git/go-git/v6 v6.0.0-20250819122726-39261590f7f3
 	github.com/ldez/go-git-cmd-wrapper/v2 v2.6.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.15.0
 	github.com/satori/go.uuid v1.2.0
-	github.com/sirupsen/logrus v1.9.0
+	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.15.0
-	github.com/stretchr/testify v1.8.3
+	github.com/stretchr/testify v1.10.0
 	github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f
-	golang.org/x/crypto v0.36.0
-	golang.org/x/exp v0.0.0-20230519143937-03e91628a987
+	golang.org/x/crypto v0.41.0
+	golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b
 	gorm.io/datatypes v1.2.0
 	gorm.io/driver/postgres v1.5.0
 	gorm.io/gorm v1.25.0
 )
 
 require (
+	dario.cat/mergo v1.0.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.3.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
@@ -45,16 +50,22 @@ require (
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-git/gcfg/v2 v2.0.2 // indirect
+	github.com/go-git/go-billy/v6 v6.0.0-20250627091229-31e2a16eef30 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.14.0 // indirect
 	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -64,6 +75,7 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -72,24 +84,25 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pjbgf/sha1cd v0.4.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/sync v0.12.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect