Skip to content

Commit 2134f12

Browse files
raphaelcoefficraphaelcoeffic
committed
feat: download targets definition from URL
Also adds automatic refresh.
1 parent 4d36e93 commit 2134f12

File tree

4 files changed

+107
-65
lines changed

4 files changed

+107
-65
lines changed

‎cmd/ebuild/run/cmd.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,19 @@ func (s *serverRunner) runAPI(cmd *cobra.Command, args []string) {
4040
fmt.Printf("failed to migrate database: %s", err)
4141
os.Exit(1)
4242
}
43-
if defs, err := targets.ReadTargetsDef("./targets.json", s.opts.SourceRepository); err != nil {
43+
if defs, err := targets.ReadTargetsDef(
44+
s.opts.TargetsDef,
45+
s.opts.SourceRepository,
46+
); err != nil {
4447
fmt.Printf("failed to read targets: %s", err)
4548
os.Exit(1)
4649
} else {
4750
targets.SetTargets(defs)
4851
}
49-
go targets.Updater(time.Minute*5, s.opts.SourceRepository)
52+
go targets.Updater(
53+
time.Second*time.Duration(s.opts.TargetsRefreshInterval),
54+
s.opts.SourceRepository,
55+
)
5056
art, err := artifactory.NewFromConfig(s.ctx, s.opts)
5157
if err != nil {
5258
fmt.Printf("failed to create artifactory: %s", err)
@@ -133,7 +139,6 @@ func NewAPICommand(s *serverRunner) *cobra.Command {
133139

134140
func NewWorkerCommand(s *serverRunner) *cobra.Command {
135141
cmd := s.makeCmd("worker", "Run a cloudbuild worker", s.runWorker)
136-
s.opts.BindWorkerOpts(cmd)
137142
return cmd
138143
}
139144

‎config/options.go‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ func LogLevelDecodeHookFunc() mapstructure.DecodeHookFunc {
5050
return func(
5151
f reflect.Type,
5252
t reflect.Type,
53-
data interface{},
54-
) (interface{}, error) {
53+
data any,
54+
) (any, error) {
5555
if f.Kind() != reflect.String {
5656
return data, nil
5757
}
@@ -85,6 +85,10 @@ type CloudbuildOpts struct {
8585
DatabaseDSN string `mapstructure:"database-dsn"`
8686
DatabaseHost string `mapstructure:"database-host"`
8787

88+
// Targets options:
89+
TargetsDef string `mapstructure:"targets"`
90+
TargetsRefreshInterval uint32 `mapstructure:"targets-refresh-interval"`
91+
8892
// Build options:
8993
BuildImage string `mapstructure:"build-img"`
9094
SourceRepository string `mapstructure:"src-repo"`
@@ -104,14 +108,16 @@ type CloudbuildOpts struct {
104108

105109
func NewOpts(v *viper.Viper) *CloudbuildOpts {
106110
return &CloudbuildOpts{
107-
Viper: v,
108-
LogLevel: InfoLevel,
109-
HTTPBindPort: 3000,
110-
BuildImage: "ghcr.io/edgetx/edgetx-builder",
111-
SourceRepository: "https://github.com/EdgeTX/edgetx.git",
112-
DownloadURL: "http://localhost:3000",
113-
StorageType: "FILE_SYSTEM_STORAGE",
114-
StoragePath: "/tmp",
111+
Viper: v,
112+
LogLevel: InfoLevel,
113+
HTTPBindPort: 3000,
114+
TargetsDef: "file:///targets.json",
115+
TargetsRefreshInterval: 300,
116+
BuildImage: "ghcr.io/edgetx/edgetx-builder",
117+
SourceRepository: "https://github.com/EdgeTX/edgetx.git",
118+
DownloadURL: "http://localhost:3000",
119+
StorageType: "FILE_SYSTEM_STORAGE",
120+
StoragePath: "/tmp",
115121
}
116122
}
117123

@@ -162,20 +168,28 @@ func (o *CloudbuildOpts) BindBuildOpts(c *cobra.Command) {
162168
c.PersistentFlags().StringVar(
163169
&o.BuildImage, "build-img", o.BuildImage, "Build docker image",
164170
)
165-
}
166-
167-
func (o *CloudbuildOpts) BindAPIOpts(c *cobra.Command) {
168-
c.Flags().Uint16VarP(&o.HTTPBindPort, "port", "p", o.HTTPBindPort, "HTTP listen port")
169-
c.Flags().IPVarP(&o.HTTPBindAddress, "listen-ip", "l", net.IPv4zero, "HTTP listen IP")
170-
c.Flags().StringVarP(
171-
&o.DownloadURL, "download-url", "u", o.DownloadURL, "Artifact download URL")
172171
c.Flags().StringVar(
173-
&o.SourceRepository, "src-repo", o.SourceRepository, "Source repository")
172+
&o.SourceRepository, "src-repo", o.SourceRepository, "Source repository",
173+
)
174174
}
175175

176-
func (o *CloudbuildOpts) BindWorkerOpts(c *cobra.Command) {
176+
func (o *CloudbuildOpts) BindAPIOpts(c *cobra.Command) {
177+
c.Flags().Uint16VarP(
178+
&o.HTTPBindPort, "port", "p", o.HTTPBindPort, "HTTP listen port",
179+
)
180+
c.Flags().IPVarP(
181+
&o.HTTPBindAddress, "listen-ip", "l", net.IPv4zero, "HTTP listen IP",
182+
)
177183
c.Flags().StringVar(
178-
&o.SourceRepository, "src-repo", o.SourceRepository, "Source repository")
184+
&o.TargetsDef, "targets", o.TargetsDef, "Targets definition",
185+
)
186+
c.Flags().Uint32Var(
187+
&o.TargetsRefreshInterval, "targets-refresh-interval",
188+
o.TargetsRefreshInterval, "Targets refresh interval",
189+
)
190+
c.Flags().StringVarP(
191+
&o.DownloadURL, "download-url", "u", o.DownloadURL, "Artifact download URL",
192+
)
179193
}
180194

181195
func (o *CloudbuildOpts) Unmarshal() error {

‎targets/targets_def.go‎

Lines changed: 54 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import (
44
"encoding/json"
55
"errors"
66
"fmt"
7+
"io"
78
"math"
9+
"net/http"
10+
"net/url"
811
"os"
912
"strings"
1013
"sync/atomic"
@@ -24,8 +27,9 @@ var (
2427
// absurdly high version.
2528
NightlyVersion = semver.New(math.MaxUint64, 0, 0, "", "")
2629

27-
ErrMissingSHA = errors.New("missing SHA")
28-
ErrMissingRef = errors.New("missing ref")
30+
ErrMissingSHA = errors.New("missing SHA")
31+
ErrMissingRef = errors.New("missing ref")
32+
ErrInvalidSchema = errors.New("invalid URL schema")
2933
)
3034

3135
type RemoteAPI struct {
@@ -71,6 +75,8 @@ type TargetsDef struct {
7175
OptionFlags OptionFlags `json:"flags"`
7276
Tags map[string]TagDef `json:"tags"`
7377
Targets map[string]*Target `json:"targets"`
78+
sourceURL string
79+
update bool
7480
}
7581

7682
func ReadTargetsDefFromBytes(data []byte, repoURL string) (*TargetsDef, error) {
@@ -84,12 +90,42 @@ func ReadTargetsDefFromBytes(data []byte, repoURL string) (*TargetsDef, error) {
8490
return &defs, nil
8591
}
8692

87-
func ReadTargetsDef(path, repoURL string) (*TargetsDef, error) {
88-
bytes, err := os.ReadFile(path)
93+
func ReadTargetsDef(targetsURL, repoURL string) (*TargetsDef, error) {
94+
src, err := url.Parse(targetsURL)
8995
if err != nil {
9096
return nil, err
9197
}
92-
return ReadTargetsDefFromBytes(bytes, repoURL)
98+
var (
99+
bytes []byte
100+
update bool
101+
)
102+
switch src.Scheme {
103+
case "", "file":
104+
log.Debugf("Reading target definitions from file: %s", src.Path)
105+
bytes, err = os.ReadFile(src.Path)
106+
if err != nil {
107+
return nil, err
108+
}
109+
case "http", "https":
110+
log.Debugf("Reading target definitions from URL: %s", src.String())
111+
if resp, err := http.Get(src.String()); err != nil {
112+
return nil, err
113+
} else {
114+
defer resp.Body.Close()
115+
if bytes, err = io.ReadAll(resp.Body); err != nil {
116+
return nil, err
117+
}
118+
update = true
119+
}
120+
default:
121+
return nil, ErrInvalidSchema
122+
}
123+
defs, err := ReadTargetsDefFromBytes(bytes, repoURL)
124+
if defs != nil {
125+
defs.sourceURL = targetsURL
126+
defs.update = update
127+
}
128+
return defs, err
93129
}
94130

95131
func NewVersionRef(v string) (*VersionRef, error) {
@@ -157,14 +193,17 @@ func (def *TargetsDef) UnmarshalJSON(text []byte) error {
157193
}
158194

159195
func (def *TargetsDef) validateSHA(repoURL string) error {
196+
return def.updateRefs(repoURL, true)
197+
}
198+
199+
func (def *TargetsDef) updateRefs(repoURL string, failEarly bool) error {
160200
var (
161201
tags map[string]string
162202
err error
163203
)
164204

165-
log.Debugf("Repository URL: %s", repoURL)
166205
if repoURL != "" {
167-
log.Debugf("Listing tags...")
206+
log.Debugf("Listing tags from %s", repoURL)
168207
tags, err = ListTags(repoURL)
169208
if err != nil {
170209
return fmt.Errorf("could not list tags from %s: %w", repoURL, err)
@@ -175,15 +214,17 @@ func (def *TargetsDef) validateSHA(repoURL string) error {
175214

176215
for k := range def.Releases {
177216
v := def.Releases[k]
178-
if v.SHA == "" {
217+
if v.SHA == "" || v.update {
179218
tag := k.String()
180-
sha, ok := tags[tag]
181-
if !ok || (sha == "") {
219+
if sha, ok := tags[tag]; ok {
220+
v.SHA = sha
221+
v.update = true
222+
log.Debugf("%s -> %s", tag, v.SHA)
223+
} else if failEarly {
182224
return fmt.Errorf("%s: %w", tag, ErrMissingSHA)
225+
} else {
226+
log.Errorf("could not update %s from %s", tag, repoURL)
183227
}
184-
v.SHA = sha
185-
v.update = true
186-
log.Debugf("%s -> %s", tag, v.SHA)
187228
}
188229
}
189230

‎targets/updater.go‎

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,42 +6,24 @@ import (
66
log "github.com/sirupsen/logrus"
77
)
88

9-
func UpdateRefs(defs *TargetsDef, repoURL string) {
10-
var (
11-
tags map[string]string
12-
err error
13-
)
14-
15-
if repoURL != "" {
16-
log.Debugf("Listing tags...")
17-
tags, err = ListTags(repoURL)
9+
func updateDefsSingleton(repoURL string) {
10+
defs := targetsDef.Load()
11+
if defs.update {
12+
newDefs, err := ReadTargetsDef(defs.sourceURL, repoURL)
1813
if err != nil {
19-
log.Errorf("could not list tags from %s: %s", repoURL, err)
14+
log.Errorf("could not update targets: %s", err)
15+
} else {
16+
targetsDef.Store(newDefs)
2017
}
2118
} else {
22-
tags = make(map[string]string)
23-
}
24-
25-
for k := range defs.Releases {
26-
v := defs.Releases[k]
27-
if v.update {
28-
tag := k.String()
29-
if sha, ok := tags[tag]; ok {
30-
v.SHA = sha
31-
log.Debugf("%s -> %s", tag, v.SHA)
32-
} else {
33-
log.Errorf("could not update %s from %s", tag, repoURL)
34-
}
19+
newDefs := *defs
20+
if err := newDefs.updateRefs(repoURL, false); err != nil {
21+
log.Errorf("could not update targets: %s", err)
3522
}
23+
targetsDef.Store(&newDefs)
3624
}
3725
}
3826

39-
func updateDefsSingleton(repoURL string) {
40-
newDefs := *targetsDef.Load()
41-
UpdateRefs(&newDefs, repoURL)
42-
targetsDef.Store(&newDefs)
43-
}
44-
4527
func Updater(interval time.Duration, repoURL string) {
4628
for {
4729
time.Sleep(interval)

0 commit comments

Comments
 (0)