Skip to content

Commit 2edd016

Browse files
europauleriknordmark
europaul
authored and
eriknordmark
committed
devUpload should not be parsed by egde-view
Since keepSentQueue contains the same or more logs than devUpload, it's enough to only use those logs when searching with edge-view. Signed-off-by: Paul Gaiduk <[email protected]>
1 parent 4bb28ac commit 2edd016

File tree

148 files changed

+178518
-1728
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

148 files changed

+178518
-1728
lines changed

‎pkg/edgeview/go.mod

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ require (
77
github.com/fsnotify/fsnotify v1.6.0
88
github.com/gorilla/websocket v1.5.0
99
github.com/grandcat/zeroconf v1.0.0
10-
github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff
11-
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241011162601-efd973923722
10+
github.com/lf-edge/eve-api/go v0.0.0-20241126174545-53f139cd5166
11+
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241126215939-16062e81d553
12+
github.com/onsi/gomega v1.29.0
1213
github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76
1314
github.com/shirou/gopsutil v3.21.11+incompatible
1415
github.com/sirupsen/logrus v1.9.3
@@ -41,4 +42,5 @@ require (
4142
golang.org/x/text v0.14.0 // indirect
4243
google.golang.org/protobuf v1.33.0 // indirect
4344
gopkg.in/yaml.v2 v2.4.0 // indirect
45+
gopkg.in/yaml.v3 v3.0.1 // indirect
4446
)

‎pkg/edgeview/go.sum

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
1313
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
1414
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
1515
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
16+
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
17+
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
1618
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
1719
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
1820
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
@@ -23,10 +25,14 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
2325
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
2426
github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
2527
github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
28+
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
29+
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
2630
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
2731
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
2832
github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
2933
github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
34+
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
35+
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
3036
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
3137
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
3238
github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
@@ -37,15 +43,17 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
3743
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
3844
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
3945
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
40-
github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff h1:3uGTOvWQFQkIrlkFalmzUmXINnzmVOAn5Zx0ryBSzxQ=
41-
github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA=
46+
github.com/lf-edge/eve-api/go v0.0.0-20241126174545-53f139cd5166 h1:TDCm5zpp6q3pZz0zO4us3cZFtzm1Tk8JIDqBOioFy1k=
47+
github.com/lf-edge/eve-api/go v0.0.0-20241126174545-53f139cd5166/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA=
4248
github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo=
4349
github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d/go.mod h1:Nn3juMJJ1G8dyHOebdZyS4jOB/fuxAd5fIajBaWjHr8=
44-
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241011162601-efd973923722 h1:wy0+46dsTULgAqVMmumzT1i2HVSMXSWeT76DuR+Il0M=
45-
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241011162601-efd973923722/go.mod h1:5zB+wlsiTBE2N/Jb0Zr7DaUJMG2OCcAP6X+dyftIdzo=
50+
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241126215939-16062e81d553 h1:89ZRk9plljAhGOkXj1Rgjg0Y0qbllax1QWAQMqUk1q8=
51+
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241126215939-16062e81d553/go.mod h1:1NZojBfxi51+dZrJzeUkTIBUEMlncjNbjnWabPXmIS4=
4652
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
4753
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
4854
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
55+
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
56+
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
4957
github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
5058
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
5159
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -122,6 +130,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
122130
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
123131
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
124132
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
133+
golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
134+
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
125135
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
126136
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
127137
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=

‎pkg/edgeview/src/edgeview_test.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright (c) 2024 Zededa, Inc.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package main
5+
6+
import (
7+
"path"
8+
"testing"
9+
"time"
10+
11+
. "github.com/onsi/gomega"
12+
)
13+
14+
func TestWalkLogDirs(t *testing.T) {
15+
g := NewWithT(t)
16+
17+
// test the walkLogDirs function
18+
newlogDir = "../../newlog/testdata"
19+
20+
const layout = "2006-01-02 15:04:05.000 -0700 MST"
21+
timestamp := "2024-11-13 10:58:52.618 +0100 CET"
22+
parsedTime, err := time.Parse(layout, timestamp)
23+
g.Expect(err).NotTo(HaveOccurred(), "failed to parse timestamp")
24+
25+
from := parsedTime.Add(-1 * time.Second)
26+
to := parsedTime.Add(1 * time.Second)
27+
foundFiles := walkLogDirs(to.Unix(), from.Unix())
28+
g.Expect(foundFiles).To(HaveLen(1), "expected exactly one file to be found")
29+
30+
expected := logfiletime{
31+
filepath: path.Join(newlogDir, "keepSentQueue/dev.log.keep.1731491932618.gz"),
32+
filesec: 1731491932,
33+
}
34+
g.Expect(foundFiles[0]).To(Equal(expected))
35+
}

‎pkg/edgeview/src/log-search.go

Lines changed: 81 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,20 @@ import (
1010
"fmt"
1111
"io"
1212
"os"
13+
"path"
1314
"path/filepath"
1415
"sort"
1516
"strconv"
1617
"strings"
1718
"time"
1819

1920
"github.com/lf-edge/eve-api/go/logs"
21+
"github.com/lf-edge/eve/pkg/pillar/types"
22+
)
23+
24+
var (
25+
newlogDir = "/persist/newlog"
26+
collectDir = newlogDir + "/collect"
2027
)
2128

2229
type logfiletime struct {
@@ -75,7 +82,7 @@ func runLogSearch(cmds cmdOpt) {
7582
return
7683
}
7784

78-
gfiles := walkLogDirs(t1, t2, now)
85+
gfiles := walkLogDirs(t1, t2)
7986
prog1 := "zcat"
8087
prog2 := "grep"
8188
arg2 := []string{"-E", pattern}
@@ -106,66 +113,52 @@ func runLogSearch(cmds cmdOpt) {
106113
fmt.Println()
107114
}
108115

109-
func walkLogDirs(t1, t2, now int64) []logfiletime {
116+
func walkLogDirs(toTimestamp, fromTimestamp int64) []logfiletime {
110117
var getfiles []logfiletime
111118

112-
files, err := os.ReadDir("/persist/newlog")
119+
subdirs, err := os.ReadDir(newlogDir)
113120
if err != nil {
114-
fmt.Printf("read /persist/newlog error %v\n", err)
121+
fmt.Printf("read %s error %v\n", newlogDir, err)
115122
return getfiles
116123
}
117124

118-
gzfiles := make(map[string][]string)
119-
for _, dir := range files {
120-
if !dir.IsDir() {
121-
continue
122-
}
123-
if strings.Contains(dir.Name(), "collect") || strings.Contains(dir.Name(), "panic") {
124-
continue
125-
}
126-
if strings.Contains(dir.Name(), "devUpload") && querytype == "app" {
127-
continue
128-
}
129-
if strings.Contains(dir.Name(), "appUpload") && querytype == "dev" {
125+
excludeDirs := []string{"collect", "panic", "devUpload"}
126+
if querytype == "dev" {
127+
excludeDirs = append(excludeDirs, "appUpload")
128+
}
129+
excludeFiles := []string{}
130+
if querytype == "app" {
131+
excludeFiles = append(excludeFiles, "dev")
132+
}
133+
if querytype == "dev" {
134+
excludeFiles = append(excludeFiles, "app")
135+
}
136+
137+
for _, dir := range subdirs {
138+
if filterDir(dir, excludeDirs) {
130139
continue
131140
}
132-
files1, err := os.ReadDir("/persist/newlog/" + dir.Name())
141+
142+
files, err := os.ReadDir(path.Join(newlogDir, dir.Name()))
133143
if err != nil {
144+
fmt.Printf("read %s error %v\n", path.Join(newlogDir, dir.Name()), err)
134145
continue
135146
}
136-
var groupfiles []string
137-
for _, f := range files1 {
138-
info, err := f.Info()
139-
if err != nil {
140-
continue
141-
}
142-
if info.ModTime().Unix() > t1 || info.ModTime().Unix() < t2 {
147+
for _, f := range files {
148+
if filterFile(f, excludeFiles) {
143149
continue
144150
}
145-
groupfiles = append(groupfiles, f.Name())
146-
}
147-
gzfiles["/persist/newlog/"+dir.Name()] = groupfiles
148-
}
149151

150-
for k, g := range gzfiles {
151-
for _, l := range g {
152-
if !strings.Contains(l, "dev") && !strings.Contains(l, "app") {
153-
continue
154-
}
155-
if querytype == "app" && !strings.Contains(l, "app") {
156-
continue
157-
}
158-
if querytype == "dev" && !strings.Contains(l, "dev") {
159-
continue
160-
}
161-
ftime := getFileTime(l)
162-
if ftime == 0 {
152+
timestamp, err := types.GetTimestampFromGzipName(f.Name())
153+
if err != nil {
163154
continue
164155
}
165-
if ftime >= t2 && ftime <= t1 {
166-
file1 := strings.TrimPrefix(l, "./")
156+
ftime := timestamp.Unix() // convert to seconds
157+
158+
if ftime >= fromTimestamp && ftime <= toTimestamp {
159+
file1 := strings.TrimPrefix(f.Name(), "./")
167160
gfile := logfiletime{
168-
filepath: k + "/" + file1,
161+
filepath: path.Join(newlogDir, dir.Name(), file1),
169162
filesec: ftime,
170163
}
171164
getfiles = append(getfiles, gfile)
@@ -180,26 +173,60 @@ func walkLogDirs(t1, t2, now int64) []logfiletime {
180173
return getfiles
181174
}
182175

183-
func searchLiveLogs(pattern string, now int64, typeStr string, idx *int, logjson bool) {
184-
files, err := os.ReadDir("/persist/newlog/collect")
185-
if err != nil {
186-
fmt.Printf("read /persist/newlog/collect error %v\n", err)
187-
return
176+
func filterDir(dir os.DirEntry, filter []string) bool {
177+
if !dir.IsDir() {
178+
return true
188179
}
180+
for _, name := range filter {
181+
if strings.Contains(dir.Name(), name) {
182+
return true
183+
}
184+
}
185+
return false
186+
}
189187

190-
for _, l := range files {
191-
if !strings.HasPrefix(l.Name(), typeStr) {
192-
continue
188+
func filterFile(file os.DirEntry, filter []string) bool {
189+
if file.IsDir() {
190+
return true
191+
}
192+
for _, name := range filter {
193+
if strings.Contains(file.Name(), name) {
194+
return true
193195
}
194-
file := "/persist/newlog/collect/" + l.Name()
196+
}
197+
return false
198+
}
199+
200+
func searchLiveLogs(pattern string, now int64, typeStr string, idx *int, logjson bool) {
201+
var filesToGrep []string
202+
switch typeStr {
203+
case "dev":
204+
filesToGrep = append(filesToGrep, path.Join(collectDir, "current.device.log"))
205+
case "app":
206+
files, err := os.ReadDir(collectDir)
207+
if err != nil {
208+
fmt.Printf("searchLiveLogs: read %s: error %v\n", collectDir, err)
209+
return
210+
}
211+
212+
for _, l := range files {
213+
if strings.HasPrefix(l.Name(), "app") {
214+
filesToGrep = append(filesToGrep, path.Join(collectDir, l.Name()))
215+
}
216+
}
217+
default:
218+
fmt.Printf("searchLiveLogs: invalid typeStr %v\n", typeStr)
219+
}
220+
221+
for _, file := range filesToGrep {
195222
searchCurrentLogs(pattern, file, typeStr, now, idx, logjson)
196223
}
197224
}
198225

199226
func searchCurrentLogs(pattern, path, typeStr string, now int64, idx *int, logjson bool) {
200227
contents, err := os.ReadFile(path)
201228
if err != nil {
202-
fmt.Printf("read %s file error: %v\n", path, err)
229+
fmt.Printf("searchCurrentLogs: read %s file error: %v\n", path, err)
203230
}
204231
lines := bytes.SplitAfter(contents, []byte("\n"))
205232
var selectlines string

‎pkg/edgeview/src/system.go

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -160,10 +160,12 @@ func getLogStats() {
160160
isDev = true
161161
}
162162

163-
time1 := getFileTime(l.Name())
164-
if time1 == 0 {
163+
timestamp, err := types.GetTimestampFromGzipName(l.Name())
164+
if err != nil {
165165
continue
166166
}
167+
time1 := timestamp.Unix() // convert to seconds
168+
167169
if isDev && (tmin == 0 || tmin > time1) {
168170
tmin = time1
169171
}
@@ -308,22 +310,6 @@ func du(currentPath string, info os.FileInfo) int64 {
308310
return size
309311
}
310312

311-
func getFileTime(filename string) int64 {
312-
var fn []string
313-
if strings.Contains(filename, ".gz") && strings.Contains(filename, ".log.") {
314-
fn = strings.Split(filename, ".gz")
315-
}
316-
if len(fn) < 2 {
317-
return 0
318-
}
319-
fn = strings.Split(fn[0], ".log.")
320-
if len(fn) < 2 {
321-
return 0
322-
}
323-
filetime, _ := strconv.Atoi(fn[1])
324-
return int64(filetime / 1000)
325-
}
326-
327313
func getVolume() {
328314
jfiles, err := listJSONFiles("/run/zedagent/AppInstanceConfig")
329315
if err != nil {

0 commit comments

Comments
 (0)