Merge branch 'lf-edge:master' into master

Author: rene

.github/workflows/publish.yml

@@ -25,13 +25,13 @@ jobs:
           - os: ubuntu-22.04
             arch: amd64
             platform: "generic"
-          - os: arm64-secure
+          - os: buildjet-4vcpu-ubuntu-2204-arm
             arch: arm64
             platform: "generic"
-          - os: arm64-secure
+          - os: buildjet-4vcpu-ubuntu-2204-arm
             arch: arm64
             platform: "nvidia-jp5"
-          - os: arm64-secure
+          - os: buildjet-4vcpu-ubuntu-2204-arm
             arch: arm64
             platform: "nvidia-jp6"
           - os: ubuntu-latest

.github/workflows/yetus.yml

@@ -25,7 +25,7 @@ jobs:
           fetch-depth: 0
 
       - name: Yetus
-        uses: apache/yetus-test-patch-action@0.15.0
+        uses: apache/yetus-test-patch-action@0.15.1
         with:
           basedir: ./src
           buildtool: nobuild

.gitignore

@@ -28,3 +28,4 @@ tools/get-deps/get-deps
 tools/get-deps/vendor
 pkg/installer/target
 pkg/installer/vendor
+pkg/monitor/eve-monitor-rs

CONTRIBUTING.md

@@ -281,6 +281,65 @@ title".
 
 Example of such a PR: [#4527](https://github.com/lf-edge/eve/pull/4527)
 
+## Kernel development
+
+The [eve-kernel](https://github.com/lf-edge/eve-kernel) project provides
+all kernel versions used by EVE. Some devices, like those from NVIDIA
+Jetson platform, use a different kernel from the main one. Each supported
+kernel is provided in a dedicated branch of the eve-kernel project so it
+facilitates the integration with EVE's build system. When contributing with
+a kernel change (adding a driver to the default configuration, providing a
+driver fix, etc) developers must pay attention to provide the change for
+all relevant kernel versions. All active branches of the eve-kernel project
+can be listed using the script `./tools/update_kernel_commits.py`:
+
+```sh
+./tools/update_kernel_commits.py -v
+```
+
+The output should be something like the following:
+
+```text
+Fetching branch information from github...
+  eve-kernel-amd64-v6.1.111-rt, Commit: c708a17493f1
+  eve-kernel-amd64-v6.1.112-generic, Commit: 272f44dbfe09
+  eve-kernel-arm64-v5.10.192-nvidia-jp5, Commit: 6e54f05fbd3b
+  eve-kernel-arm64-v5.15.136-nvidia-jp6, Commit: 22e03b8516f2
+  eve-kernel-arm64-v6.1.112-generic, Commit: 9f160b774dbc
+  eve-kernel-riscv64-v6.1.112-generic, Commit: 18e1d313b90b
+    skipping: linux-6.1.y
+    skipping: main
+    skipping: rucoder/new-sbom
+Checking for updated branches...
+  eve-kernel-amd64-v6.1.111-rt, current commit: c708a17493f1
+  eve-kernel-amd64-v6.1.112-generic, current commit: 272f44dbfe09
+  eve-kernel-arm64-v5.10.192-nvidia-jp5, current commit: 6e54f05fbd3b
+  eve-kernel-arm64-v5.15.136-nvidia-jp6, current commit: 22e03b8516f2
+  eve-kernel-arm64-v6.1.112-generic, current commit: 9f160b774dbc
+  eve-kernel-riscv64-v6.1.112-generic, current commit: 18e1d313b90b
+Checking for removed branches...
+  eve-kernel-amd64-v6.1.111-rt, current commit: c708a17493f1
+  eve-kernel-amd64-v6.1.112-generic, current commit: 272f44dbfe09
+  eve-kernel-arm64-v5.10.192-nvidia-jp5, current commit: 6e54f05fbd3b
+  eve-kernel-arm64-v5.15.136-nvidia-jp6, current commit: 22e03b8516f2
+  eve-kernel-arm64-v6.1.112-generic, current commit: 9f160b774dbc
+  eve-kernel-riscv64-v6.1.112-generic, current commit: 18e1d313b90b
+No kernel updates available.
+```
+
+In this case the active branches are:
+
+* eve-kernel-amd64-v6.1.111-rt
+* eve-kernel-amd64-v6.1.112-generic
+* eve-kernel-arm64-v5.10.192-nvidia-jp5
+* eve-kernel-arm64-v5.15.136-nvidia-jp6
+* eve-kernel-arm64-v6.1.112-generic
+* eve-kernel-riscv64-v6.1.112-generic
+
+For instance, if the developer enables an USB Webcam driver in the
+`eve-kernel-amd64-v6.1.112-generic`, it must provide the patch for all
+other active branches as well, when applicable.
+
 ## Check how we are doing
 
 Linux Foundation maintains a [project health dashboard](https://insights.lfx.linuxfoundation.org/foundation/lfedge/overview/github?project=project-eve)

Makefile

@@ -12,7 +12,7 @@ uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1)))
 
 # you are not supposed to tweak these variables -- they are effectively R/O
 HV_DEFAULT=kvm
-GOVER ?= 1.20.1
+GOVER ?= 1.24.1
 PKGBASE=github.com/lf-edge/eve
 GOMODULE=$(PKGBASE)/pkg/pillar
 GOTREE=$(CURDIR)/pkg/pillar
@@ -352,7 +352,7 @@ endif
 
 # for the go build sources
 GOSOURCES=$(BUILDTOOLS_BIN)/go-sources-and-licenses
-GOSOURCES_VERSION=6047a068b2702ed2687cd13dcb7eaa2542d20344
+GOSOURCES_VERSION=v1.0.0
 GOSOURCES_SOURCE=github.com/deitch/go-sources-and-licenses
 
 # for the compare sbom and collecte sources
@@ -458,7 +458,7 @@ currentversion:
 
 .PHONY: currentversion linuxkit pkg/kernel
 
-test: $(LINUXKIT) test-images-patches | $(DIST)
+test: $(LINUXKIT) pkg/pillar test-images-patches | $(DIST)
 	@echo Running tests on $(GOMODULE)
 	make -C pkg/pillar test
 	cp pkg/pillar/results.json $(DIST)/
@@ -491,18 +491,19 @@ $(DOCKERFILE_FROM_CHECKER): $(DOCKERFILE_FROM_CHECKER_DIR)/*.go $(DOCKERFILE_FRO
 IGNORE_DOCKERFILE_HASHES_PKGS=alpine installer
 IGNORE_DOCKERFILE_HASHES_EVE_TOOLS=bpftrace-compiler
 
+IGNORE_DOCKERFILE_DOT_GO_DIR=$(shell find .go/ -name Dockerfile -exec echo "-i {}" \;)
 IGNORE_DOCKERFILE_HASHES_PKGS_ARGS=$(foreach pkg,$(IGNORE_DOCKERFILE_HASHES_PKGS),-i pkg/$(pkg)/Dockerfile)
 IGNORE_DOCKERFILE_HASHES_EVE_TOOLS_ARGS=$(foreach tool,$(IGNORE_DOCKERFILE_HASHES_EVE_TOOLS),$(addprefix -i ,$(shell find eve-tools/$(tool) -path '*/vendor' -prune -o -name Dockerfile -print)))
 
 .PHONY: check-docker-hashes-consistency
 check-docker-hashes-consistency: $(DOCKERFILE_FROM_CHECKER)
 	@echo "Checking Dockerfiles for inconsistencies"
-	$(DOCKERFILE_FROM_CHECKER) ./ $(IGNORE_DOCKERFILE_HASHES_PKGS_ARGS) $(IGNORE_DOCKERFILE_HASHES_EVE_TOOLS_ARGS)
+	$(DOCKERFILE_FROM_CHECKER) ./ $(IGNORE_DOCKERFILE_HASHES_PKGS_ARGS) $(IGNORE_DOCKERFILE_HASHES_EVE_TOOLS_ARGS) $(IGNORE_DOCKERFILE_DOT_GO_DIR)
 
 yetus:
 	@echo Running yetus
 	mkdir -p yetus-output
-	docker run --rm -v $(CURDIR):/src:delegated,z ghcr.io/apache/yetus:0.15.0 \
+	docker run --rm -v $(CURDIR):/src:delegated,z ghcr.io/apache/yetus:0.15.1 \
 		--basedir=/src \
 		--test-parallel=true \
 		--dirty-workspace \
@@ -760,8 +761,8 @@ ifeq ($(ROOTFS_FORMAT),squash)
 endif
 	$(QUIET): $@: Succeeded
 
-$(GET_DEPS):
-	$(MAKE) -C $(GET_DEPS_DIR) GOOS=$(LOCAL_GOOS)
+$(GET_DEPS): $(GOBUILDER)
+	$(QUIET)$(DOCKER_GO) "make" $(GET_DEPS_DIR)
 
 sbom_info:
 	@echo "$(SBOM)"
@@ -913,7 +914,7 @@ cache-export-docker-load: $(LINUXKIT)
 	rm -rf ${TARFILE}
 
 %-cache-export-docker-load: $(LINUXKIT)
-	$(eval IMAGE_TAG := $(shell $(MAKE) $*-show-tag))
+	$(eval IMAGE_TAG := $(shell $(LINUXKIT) pkg show-tag --canonical pkg/$*))
 	$(eval CACHE_CONTENT := $(shell $(LINUXKIT) cache ls 2>&1))
 	$(if $(filter $(IMAGE_TAG),$(CACHE_CONTENT)),$(MAKE) cache-export-docker-load IMAGE=$(IMAGE_TAG),@echo "Missing image $(IMAGE_TAG) in cache")
 

docs/CONFIG-PROPERTIES.md

@@ -68,10 +68,14 @@
 | goroutine.leak.detection.cooldown.minutes | integer (minutes) | 5 | Cooldown period in minutes after the leak detection is triggered. During this period, no stack traces are collected; only warning messages are logged. |
 | kubevirt.drain.timeout | integer | 24 | hours to allow kubernetes to drain a node |
 | memory-monitor.enabled | boolean | false | Enable external memory monitoring and memory pressure events handling |
+| debug.tui.loglevel | string | info | Set log level for EVE Text UI (TUI) monitor. Possible values are "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE" and are case insensitive
+| log.dedup.window.size | integer | 0 | The size of the log deduplicator's sliding window (in number of messages). See logging [docs](LOGGING.md#log-filtering-counting-and-deduplication) for details. If the window size is set to 0 (default), no deduplication is performed. |
+| log.count.filenames | string | "" | Comma-separated list of log's filenames to be counted and logged once instead of logging them every time. Example `/my-pkg/main.go:123,/other-pkg/code.go:42`. See logging [docs](LOGGING.md#log-filtering-counting-and-deduplication) for details. |
+| log.filter.filenames | string | "" | Comma-separated list of log's filenames to be filtered out. Example `/my-pkg/main.go:123,/other-pkg/code.go:42`. See logging [docs](LOGGING.md#log-filtering-counting-and-deduplication) for details. |
 
 ## Log levels
 
-Log level can be set for three different components of EVE: EVE microservices, syslog, and kernel.
+Log level can be set for four different components of EVE: EVE microservices, syslog, kernel, and TUI monitor application. Logs for TUI monitor are not sent to the controller and only available locally on the device.
 The log levels set this way are used to control the verbosity of the logs produced by the corresponding components.
 All logs produced this way will be saved locally in /persist/newlog/keepSentQueue/ directory and will be subject to rotation based on the max total size of stored logs.
 
@@ -94,6 +98,7 @@ A corresponding "remote" log level can be set for each of the three components:
 | debug.syslog.remote.loglevel | string | info | level of the syslog messages sent to the controller. System default loglevel string representation should be used as described here ["https://man7.org/linux/man-pages/man3/syslog.3.html"]: emerg, alert, crit, err, warning, notice, info, debug. |
 | debug.kernel.loglevel | string | info | level of the produced kernel log messages. System default loglevel string representation should be used as described here ["https://man7.org/linux/man-pages/man3/syslog.3.html"]: emerg, alert, crit, err, warning, notice, info, debug. |
 | debug.kernel.remote.loglevel | string | info | level of the kernel log messages sent to the controller. System default loglevel string representation should be used as described here ["https://man7.org/linux/man-pages/man3/syslog.3.html"]: emerg, alert, crit, err, warning, notice, info, debug. |
+debug.tui.loglevel | string | info | Set log level for EVE Text UI (TUI) monitor. Possible values are "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE" and are case insensitive
 
 In addition, there can be per-agent settings to overwrite the default log level set for EVE microservices.
 These use the same log levels as the default log level settings (logrus).

docs/LOGGING.md

@@ -36,6 +36,28 @@ The following diagram shows the flow of logs from containers to newlogd and to c
 
 ![EVE Log Flow](images/eve_newlog_flow.png)
 
+### Application Logging and Nested Application Logging
+
+For logs originating from sources containing the string `guest_vm-[Domain-ID]`, the `newlogd` service formats each log entry and writes it to a log file. These log files are named with the prefix `app.<app-uuid>`. This naming convention allows the controller to quickly dispatch log entries to the appropriate application instance for storage and display.
+
+When log entries originate from nested applications within a runtime VM, the agent inside the runtime formats the entry using a proto/JSON structure defined as `NestedAppInstanceLogMsg`:
+
+```go
+type NestedAppInstanceLogMsg struct {
+  // NestedAppId is the AppId of the nested application instance or virtual application instance.
+  // For example, a Docker Compose application instance ID.
+  NestedAppId string `protobuf:"bytes,1,opt,name=nested_app_id,json=nestedAppId,proto3" json:"nested_app_id,omitempty"`
+  // ContainerName is the shortened name of the container.
+  // For example, a Compose application container name is project_name-service-replica.
+  // This field should contain the service-replica suffix.
+  ContainerName string `protobuf:"bytes,2,opt,name=container_name,json=containerName,proto3" json:"container_name,omitempty"`
+  // Msg is the original log message from the runtime container.
+  Msg string `protobuf:"bytes,3,opt,name=msg,proto3" json:"msg,omitempty"`
+}
+```
+
+The newlogd service subscribes to the NestedAppDomainStatus publication from zedrouter. It processes nested application logs similarly to how it handles DomainStatus publications, specifically in terms of generating log files for application log entries. If the VM's deployment type is "Docker" and a log entry's content can be successfully unmarshalled from JSON into the NestedAppInstanceLogMsg structure, the entry is written to a file with the prefix app.<nestedAppId>. This facilitates rapid log dispatch to the correct application instance (including nested application instances) on the controller side.
+
 ## Log Aggregation, Reformatting and Compression for Persistent Log Files
 
 All logs collected from various containers/services/kernel in the system will reach newlogd daemon.
@@ -88,6 +110,34 @@ There are no granularity nobs for the edge apps' log levels - all logs generated
 
 For the full list of log level parameters and the possible values, see the [config-properties](CONFIG-PROPERTIES.md#log-levels) doc.
 
+## Log filtering, counting and deduplication
+
+To reduce the amount of logs sent to the controller, EVE offers three mechanisms:
+
+* filter
+* counter
+* deduplicator
+
+All of the above are implemented in the newlogd daemon.
+The transformations are performed when newlog compresses the logs into gzip files.
+
+Filter simply removes the specified log entries from the log file.
+
+Counter counts the number of specified log entries and only keeps the first occurrence of the specified log entries.
+It also adds a "count" tag to that log entry with the number of times that log entry was repeated in the file.
+
+The identifier for the log entries to be filtered or counted is the field "Filename" that's present in every [log entry](https://github.com/lf-edge/eve-api/go/logs/log.pb.go#L39).
+It's a string that contains the name of the source code file and the line number where the log entry was generated (e.g. "/my-pkg/main.go:123").
+The config properties for the filter and counter are:
+
+* `log.count.filenames`
+* `log.filter.filenames`
+
+Deduplicator goes through all log entries in the file, keeping a sliding window of the last N error messages.
+It checks: if the N+1'th message is already in the window - it is not logged.
+Either way the first message in the window is replaced with the new message (FIFO).
+The window size is configurable via the `log.dedup.window.size` config property.
+
 ## Log export to cloud
 
 "loguploader" is a pillar service which is responsible for uploading the gzip log files to the controller. The binary data of a gzip file is the payload portion of the authentication protobuf envolope structure. This is similar to all the other EVE POST messages, except that in those messages the payload usually is data of another protobuf structure.