Merge pull request #72 from netboxlabs/develop

Release 🚚

Author: leoparente

.github/workflows/develop.yaml

@@ -24,25 +24,30 @@ jobs:
           uses: actions/checkout@v4
 
         - name: Set up QEMU
-          uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf #v3.2.0
+          uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 #v3.4.0
 
         - name: Set up Docker Buildx
-          uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 #v3.8.0
+          uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca #v3.9.0
 
         - name: Login to Docker Hub
           uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 #v3.3.0
           with:
             username: ${{ secrets.DOCKERHUB_USERNAME }}
             password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+        - name: Verify QEMU installation
+          run: |
+              docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+              docker run --rm --platform linux/arm64 debian:latest uname -m
+
         - name: Set build info
           run: |
             echo ${GITHUB_SHA::7} > ./agent/version/BUILD_COMMIT.txt
             LATEST_RELEASE=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/latest" | jq -r '.tag_name')
             echo $LATEST_RELEASE > ./agent/version/BUILD_VERSION.txt
 
         - name: Build image and push
-          uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 #v6.10.0
+          uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14.0
           with:
             context: .
             file: agent/docker/Dockerfile

.github/workflows/release.yaml

@@ -132,24 +132,29 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf #v3.2.0
+        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 #v3.4.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 #v3.8.0
+        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca #v3.9.0
 
       - name: Login to Docker Hub
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 #v3.3.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Verify QEMU installation
+        run: |
+              docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+              docker run --rm --platform linux/arm64 debian:latest uname -m
+
       - name: Set build info
         run: |
           echo $BUILD_COMMIT > ./agent/version/BUILD_COMMIT.txt
           echo $BUILD_VERSION > ./agent/version/BUILD_VERSION.txt
 
       - name: Build image and push
-        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 #v6.10.0
+        uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14.0
         with:
           context: .
           file: agent/docker/Dockerfile

README.md

@@ -24,7 +24,9 @@ orb:
   ...
 ```
 
-Currently, only the `local` manager is supported, which retrieves policies from the local configuration file passed to the agent.
+Currently, only the `local` and `git` sources are supported for config manager.
+- [Local](./docs/configs/local.md)
+- [Git](./docs/configs/git.md)
 
 ### Backends
 The `backends` section specifies what Orb agent backends should be enabled. Each Orb agent backend offers specific discovery or observability capabilities and may require specific configuration information.  

agent/agent.go

@@ -7,15 +7,14 @@ import (
 	"runtime"
 	"time"
 
-	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/google/uuid"
 	"github.com/mitchellh/mapstructure"
-	"github.com/orb-community/orb/fleet"
 	"go.uber.org/zap"
 
 	"github.com/netboxlabs/orb-agent/agent/backend"
 	"github.com/netboxlabs/orb-agent/agent/config"
-	manager "github.com/netboxlabs/orb-agent/agent/policyMgr"
+	"github.com/netboxlabs/orb-agent/agent/configmgr"
+	"github.com/netboxlabs/orb-agent/agent/policymgr"
 	"github.com/netboxlabs/orb-agent/agent/version"
 )
 
@@ -32,8 +31,6 @@ type Agent interface {
 type orbAgent struct {
 	logger            *zap.Logger
 	config            config.Config
-	client            mqtt.Client
-	agentID           string
 	backends          map[string]backend.Backend
 	backendState      map[string]*backend.State
 	backendsCommon    config.BackendCommons
@@ -42,24 +39,18 @@ type orbAgent struct {
 
 	asyncContext context.Context
 
-	hbTicker        *time.Ticker
 	heartbeatCtx    context.Context
 	heartbeatCancel context.CancelFunc
 
-	// Agent RPC channel, configured from command line
-	baseTopic        string
-	rpcFromCoreTopic string
-	heartbeatsTopic  string
-
 	// Retry Mechanism to ensure the Request is received
 	groupRequestSucceeded  context.CancelFunc
 	policyRequestSucceeded context.CancelFunc
 
 	// AgentGroup channels sent from core
 	groupsInfos map[string]groupInfo
 
-	policyManager manager.PolicyManager
-	configManager config.Manager
+	policyManager policymgr.PolicyManager
+	configManager configmgr.Manager
 }
 
 type groupInfo struct {
@@ -71,7 +62,7 @@ var _ Agent = (*orbAgent)(nil)
 
 // New creates a new agent
 func New(logger *zap.Logger, c config.Config) (Agent, error) {
-	pm, err := manager.New(logger, c)
+	pm, err := policymgr.New(logger, c)
 	if err != nil {
 		logger.Error("error during create policy manager, exiting", zap.Error(err))
 		return nil, err
@@ -80,31 +71,11 @@ func New(logger *zap.Logger, c config.Config) (Agent, error) {
 		logger.Error("policy manager failed to get repository", zap.Error(err))
 		return nil, err
 	}
-	cm := config.New(logger, c.OrbAgent.ConfigManager)
+	cm := configmgr.New(logger, pm, c.OrbAgent.ConfigManager)
 
 	return &orbAgent{logger: logger, config: c, policyManager: pm, configManager: cm, groupsInfos: make(map[string]groupInfo)}, nil
 }
 
-func (a *orbAgent) managePolicies() error {
-	if a.config.OrbAgent.Policies == nil {
-		return errors.New("no policies specified")
-	}
-
-	for beName, policy := range a.config.OrbAgent.Policies {
-		_, ok := a.backends[beName]
-		if !ok {
-			return errors.New("backend not found: " + beName)
-		}
-		for pName, data := range policy {
-			id := uuid.NewString()
-			payload := fleet.AgentPolicyRPCPayload{Action: "manage", Name: pName, DatasetID: id, Backend: beName, Version: 1, Data: data}
-			a.policyManager.ManagePolicy(payload)
-		}
-
-	}
-	return nil
-}
-
 func (a *orbAgent) startBackends(agentCtx context.Context) error {
 	a.logger.Info("registered backends", zap.Strings("values", backend.GetList()))
 	a.logger.Info("requested backends", zap.Any("values", a.config.OrbAgent.Backends))
@@ -120,7 +91,7 @@ func (a *orbAgent) startBackends(agentCtx context.Context) error {
 			return fmt.Errorf("failed to decode common backend config: %w", err)
 		}
 	}
-	commonConfig.Otel.AgentTags = a.config.OrbAgent.Tags
+	commonConfig.Otel.AgentLabels = a.config.OrbAgent.Labels
 	a.backendsCommon = commonConfig
 	delete(a.config.OrbAgent.Backends, "common")
 
@@ -171,19 +142,12 @@ func (a *orbAgent) Start(ctx context.Context, cancelFunc context.CancelFunc) err
 	a.rpcFromCancelFunc = cancelAllAsync
 	a.cancelFunction = cancelFunc
 	a.logger.Info("agent started", zap.String("version", version.GetBuildVersion()), zap.Any("routine", agentCtx.Value(routineKey)))
-	mqtt.CRITICAL = &agentLoggerCritical{a: a}
-	mqtt.ERROR = &agentLoggerError{a: a}
-
-	if a.config.OrbAgent.Debug.Enable {
-		a.logger.Info("debug logging enabled")
-		mqtt.DEBUG = &agentLoggerDebug{a: a}
-	}
 
 	if err := a.startBackends(ctx); err != nil {
 		return err
 	}
 
-	if err := a.managePolicies(); err != nil {
+	if err := a.configManager.Start(a.config, a.backends); err != nil {
 		return err
 	}
 
@@ -193,9 +157,7 @@ func (a *orbAgent) Start(ctx context.Context, cancelFunc context.CancelFunc) err
 }
 
 func (a *orbAgent) logonWithHeartbeat() {
-	a.hbTicker = time.NewTicker(HeartbeatFreq)
 	a.heartbeatCtx, a.heartbeatCancel = a.extendContext("heartbeat")
-	go a.sendHeartbeats(a.heartbeatCtx, a.heartbeatCancel)
 	a.logger.Info("heartbeat routine started")
 }
 
@@ -204,11 +166,6 @@ func (a *orbAgent) logoffWithHeartbeat(ctx context.Context) {
 	if a.heartbeatCtx != nil {
 		a.heartbeatCancel()
 	}
-	if a.client != nil && a.client.IsConnected() {
-		if token := a.client.Unsubscribe(a.rpcFromCoreTopic); token.Wait() && token.Error() != nil {
-			a.logger.Warn("failed to unsubscribe to RPC channel", zap.Error(token.Error()))
-		}
-	}
 }
 
 func (a *orbAgent) Stop(ctx context.Context) {
@@ -225,9 +182,6 @@ func (a *orbAgent) Stop(ctx context.Context) {
 		}
 	}
 	a.logoffWithHeartbeat(ctx)
-	if a.client != nil && a.client.IsConnected() {
-		a.client.Disconnect(0)
-	}
 	a.logger.Debug("stopping agent with number of go routines and go calls", zap.Int("goroutines", runtime.NumGoroutine()), zap.Int64("gocalls", runtime.NumCgoCall()))
 	if a.policyRequestSucceeded != nil {
 		a.policyRequestSucceeded()
@@ -261,7 +215,6 @@ func (a *orbAgent) RestartBackend(ctx context.Context, name string, reason strin
 		a.backendState[name].LastError = fmt.Sprintf("failed to reset backend: %v", err)
 		a.logger.Error("failed to reset backend", zap.String("backend", name), zap.Error(err))
 	}
-	be.SetCommsClient(a.agentID, &a.client, fmt.Sprintf("%s/?/%s", a.baseTopic, name))
 
 	return nil
 }

agent/backend/backend.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"time"
 
-	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"go.uber.org/zap"
 
 	"github.com/netboxlabs/orb-agent/agent/config"
@@ -49,7 +48,6 @@ func (s RunningStatus) String() string {
 // Backend is the interface that all backends must implement
 type Backend interface {
 	Configure(*zap.Logger, policies.PolicyRepo, map[string]interface{}, config.BackendCommons) error
-	SetCommsClient(string, *mqtt.Client, string)
 	Version() (string, error)
 	Start(ctx context.Context, cancelFunc context.CancelFunc) error
 	Stop(ctx context.Context) error

agent/backend/devicediscovery/device_discovery.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"time"
 
-	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/go-cmd/cmd"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
@@ -50,8 +49,6 @@ type deviceDiscoveryBackend struct {
 	statusChan <-chan cmd.Status
 	cancelFunc context.CancelFunc
 	ctx        context.Context
-
-	mqttClient *mqtt.Client
 }
 
 type info struct {
@@ -87,10 +84,6 @@ func (d *deviceDiscoveryBackend) Configure(logger *zap.Logger, repo policies.Pol
 	return nil
 }
 
-func (d *deviceDiscoveryBackend) SetCommsClient(_ string, client *mqtt.Client, _ string) {
-	d.mqttClient = client
-}
-
 func (d *deviceDiscoveryBackend) Version() (string, error) {
 	var info info
 	err := d.request("status", &info, http.MethodGet, http.NoBody, "application/json", versionTimeout)

agent/backend/networkdiscovery/network_discovery.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"time"
 
-	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/go-cmd/cmd"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
@@ -50,8 +49,6 @@ type networkDiscoveryBackend struct {
 	statusChan <-chan cmd.Status
 	cancelFunc context.CancelFunc
 	ctx        context.Context
-
-	mqttClient *mqtt.Client
 }
 
 type info struct {
@@ -87,10 +84,6 @@ func (d *networkDiscoveryBackend) Configure(logger *zap.Logger, repo policies.Po
 	return nil
 }
 
-func (d *networkDiscoveryBackend) SetCommsClient(_ string, client *mqtt.Client, _ string) {
-	d.mqttClient = client
-}
-
 func (d *networkDiscoveryBackend) Version() (string, error) {
 	var info info
 	err := d.request("status", &info, http.MethodGet, http.NoBody, "application/json", versionTimeout)