Skip to content

Commit 32d20fd

Browse files
europauleuropaul
committed
newlog: make filename parsing future-proof
Instead of parsing filenames according to a certain schema, we should use regexp to look for timestamps or UUIDs. This way the file names can change in the future without affecting backward compatibility with the existing code for parsing filenames. Signed-off-by: Paul Gaiduk <[email protected]>
1 parent 6b41528 commit 32d20fd

File tree

3 files changed

+241
-36
lines changed

3 files changed

+241
-36
lines changed

‎pkg/pillar/cmd/loguploader/loguploader.go

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,7 @@ func doFetchSend(ctx *loguploaderContext, zipDir string, iter *int) int {
639639
if !strings.HasSuffix(f.Name(), ".gz") {
640640
continue
641641
}
642-
timestamp, err := types.GetTimestampFromGzipName(f.Name())
642+
timestamp, err := types.GetTimestampFromFileName(f.Name())
643643
if err != nil {
644644
continue
645645
}
@@ -722,22 +722,14 @@ func doFetchSend(ctx *loguploaderContext, zipDir string, iter *int) int {
722722
}
723723

724724
func buildAppUUIDMap(fName string) {
725-
var appUUID string
726-
if strings.HasPrefix(fName, types.AppPrefix) && strings.HasSuffix(fName, ".gz") {
727-
fStr1 := strings.TrimPrefix(fName, types.AppPrefix)
728-
fStr := strings.Split(fStr1, types.AppSuffix)
729-
if len(fStr) != 2 {
730-
err := fmt.Errorf("app split is not 2")
731-
log.Error(err)
732-
return
733-
}
734-
appUUID = fStr[0]
725+
appUUID, err := types.GetUUIDFromFileName(fName)
726+
if err != nil {
727+
log.Errorf("buildAppUUIDMap: cannot parse app log filename %s: %v", fName, err)
728+
return
735729
}
736730

737-
if len(appUUID) > 0 {
738-
if _, ok := appGzipMap[appUUID]; !ok {
739-
appGzipMap[appUUID] = true
740-
}
731+
if _, ok := appGzipMap[appUUID]; !ok {
732+
appGzipMap[appUUID] = true
741733
}
742734
}
743735

@@ -769,13 +761,10 @@ func sendToCloud(ctx *loguploaderContext, data []byte, iter int, fName string, f
769761
var logsURL, appLogURL string
770762
var sentFailed, serviceUnavailable bool
771763
if isApp {
772-
fStr1 := strings.TrimPrefix(fName, types.AppPrefix)
773-
fStr := strings.Split(fStr1, types.AppSuffix)
774-
if len(fStr) != 2 {
775-
err := fmt.Errorf("app split is not 2")
776-
log.Fatal(err)
764+
appUUID, err := types.GetUUIDFromFileName(fName)
765+
if err != nil {
766+
return false, fmt.Errorf("sendToCloud: cannot parse app file name %s: %v", fName, err)
777767
}
778-
appUUID := fStr[0]
779768
if ctx.zedcloudCtx.V2API {
780769
appLogURL = fmt.Sprintf("apps/instanceid/%s/newlogs", appUUID)
781770
} else {

‎pkg/pillar/types/newlogtypes.go

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package types
55

66
import (
77
"fmt"
8+
"regexp"
89
"strconv"
910
"strings"
1011
"time"
@@ -85,21 +86,50 @@ type NewlogMetrics struct {
8586
AppMetrics logfileMetrics // App metrics
8687
}
8788

88-
// GetTimestampFromGzipName - get timestamp from gzip file name
89-
func GetTimestampFromGzipName(fName string) (time.Time, error) {
90-
// here are example file names:
91-
// app.6656f860-7563-4bbf-8bba-051f5942982b.log.1730464687367.gz
92-
// dev.log.keep.1730404601953.gz
93-
// dev.log.upload.1730404601953.gz
94-
// the timestamp is the number between the last two dots
95-
nameParts := strings.Split(fName, ".")
96-
if len(nameParts) < 2 {
97-
return time.Time{}, fmt.Errorf("getTimestampFromGzipName: invalid log file name %s", fName)
89+
var (
90+
timestampRegex *regexp.Regexp
91+
uuidRegex *regexp.Regexp
92+
)
93+
94+
func init() {
95+
// Regular expression to match a timestamp
96+
timestampRegex = regexp.MustCompile(`^\d+$`)
97+
98+
// UUID regex pattern (supports v4 UUIDs like "123e4567-e89b-12d3-a456-426614174000")
99+
uuidRegex = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`)
100+
}
101+
102+
// GetTimestampFromFileName extracts a millisecond timestamp from a filename
103+
func GetTimestampFromFileName(filename string) (time.Time, error) {
104+
// Split the filename into parts using dots
105+
parts := strings.Split(filename, ".")
106+
107+
// Check each part for a timestamp match
108+
for _, part := range parts {
109+
if timestampRegex.MatchString(part) {
110+
// Convert the matched timestamp string to an integer
111+
timestamp, err := strconv.ParseInt(part, 10, 64)
112+
if err != nil {
113+
return time.Time{}, fmt.Errorf("failed to parse timestamp: %s", err)
114+
}
115+
return time.Unix(0, timestamp*int64(time.Millisecond)), nil // Return the first valid timestamp found
116+
}
98117
}
99-
timeStr := nameParts[len(nameParts)-2]
100-
fTime, err := strconv.Atoi(timeStr)
101-
if err != nil {
102-
return time.Time{}, fmt.Errorf("getTimestampFromGzipName: %w", err)
118+
119+
return time.Time{}, fmt.Errorf("no timestamp found in filename: %s", filename)
120+
}
121+
122+
// GetUUIDFromFileName extracts a UUID from a filename with dot-delimited parts
123+
func GetUUIDFromFileName(filename string) (string, error) {
124+
// Split the filename into parts using dots
125+
parts := strings.Split(filename, ".")
126+
127+
// Check each part for a UUID match
128+
for _, part := range parts {
129+
if uuidRegex.MatchString(part) {
130+
return part, nil // Return the first UUID found
131+
}
103132
}
104-
return time.Unix(0, int64(fTime)*int64(time.Millisecond)), nil
133+
134+
return "", fmt.Errorf("no UUID found in filename: %s", filename)
105135
}

‎pkg/pillar/types/newlogtypes_test.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// Copyright (c) 2025 Zededa, Inc.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package types
5+
6+
import (
7+
"testing"
8+
"time"
9+
10+
"github.com/onsi/gomega"
11+
)
12+
13+
func TestGetTimestampFromFileName(t *testing.T) {
14+
t.Parallel()
15+
g := gomega.NewWithT(t)
16+
17+
tests := []struct {
18+
name string
19+
filename string
20+
wantTime time.Time
21+
wantError bool
22+
}{
23+
{
24+
name: "Valid timestamp in filename",
25+
filename: "dev.log.1731491904032.gz",
26+
wantTime: time.Unix(0, 1731491904032*int64(time.Millisecond)),
27+
wantError: false,
28+
},
29+
{
30+
name: "Valid timestamp in regular filename",
31+
filename: "dev.log.1731491904032",
32+
wantTime: time.Unix(0, 1731491904032*int64(time.Millisecond)),
33+
wantError: false,
34+
},
35+
{
36+
name: "Valid timestamp with UUID",
37+
filename: "app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496.gz",
38+
wantTime: time.Unix(0, 1731935033496*int64(time.Millisecond)),
39+
wantError: false,
40+
},
41+
{
42+
name: "Two timestamps in filename",
43+
filename: "dev.log.1731935033496.123.gz",
44+
wantTime: time.Unix(0, 1731935033496*int64(time.Millisecond)),
45+
wantError: false,
46+
},
47+
{
48+
name: "Invalid timestamp in filename",
49+
filename: "dev.log.invalidtimestamp.gz",
50+
wantTime: time.Time{},
51+
wantError: true,
52+
},
53+
{
54+
name: "No timestamp in filename",
55+
filename: "dev.log.gz",
56+
wantTime: time.Time{},
57+
wantError: true,
58+
},
59+
{
60+
name: "Old timestamp (short format) in filename",
61+
filename: "dev.log.123.gz",
62+
wantTime: time.Unix(0, 123*int64(time.Millisecond)),
63+
wantError: false,
64+
},
65+
{
66+
name: "Old timestamp (long format) in filename",
67+
filename: "dev.log.0000000000123.gz",
68+
wantTime: time.Unix(0, 123*int64(time.Millisecond)),
69+
wantError: false,
70+
},
71+
}
72+
73+
for _, tt := range tests {
74+
tt := tt // create a new variable to hold the value of tt to avoid being overwritten by the next iteration (needed until Go 1.23)
75+
t.Run(tt.name, func(t *testing.T) {
76+
t.Parallel()
77+
gotTime, err := GetTimestampFromFileName(tt.filename)
78+
if tt.wantError {
79+
g.Expect(err).To(gomega.HaveOccurred())
80+
} else {
81+
g.Expect(err).NotTo(gomega.HaveOccurred())
82+
g.Expect(gotTime).To(gomega.Equal(tt.wantTime))
83+
}
84+
})
85+
}
86+
}
87+
88+
func FuzzGetTimestampFromFileName(f *testing.F) {
89+
testcases := []string{
90+
"dev.log.1731491904032.gz",
91+
"app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496.gz",
92+
"dev.log.invalidtimestamp.gz",
93+
"dev.log.gz",
94+
"dev.log.123456789012.gz",
95+
"dev.log.1234567890123456.gz",
96+
}
97+
98+
for _, tc := range testcases {
99+
f.Add(tc)
100+
}
101+
102+
f.Fuzz(func(t *testing.T, filename string) {
103+
_, _ = GetTimestampFromFileName(filename)
104+
})
105+
}
106+
107+
func TestGetUUIDFromFileName(t *testing.T) {
108+
t.Parallel()
109+
g := gomega.NewWithT(t)
110+
111+
tests := []struct {
112+
name string
113+
filename string
114+
wantUUID string
115+
wantError bool
116+
}{
117+
{
118+
name: "Valid UUID in filename",
119+
filename: "app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496.gz",
120+
wantUUID: "8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d",
121+
wantError: false,
122+
},
123+
{
124+
name: "Valid UUID in regular filename",
125+
filename: "app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496",
126+
wantUUID: "8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d",
127+
wantError: false,
128+
},
129+
{
130+
name: "Valid UUID with timestamp",
131+
filename: "app.123e4567-e89b-12d3-a456-426614174000.log.1731935033496.gz",
132+
wantUUID: "123e4567-e89b-12d3-a456-426614174000",
133+
wantError: false,
134+
},
135+
{
136+
name: "No UUID in filename",
137+
filename: "dev.log.1731491904032.gz",
138+
wantUUID: "",
139+
wantError: true,
140+
},
141+
{
142+
name: "Invalid UUID in filename",
143+
filename: "app.invalid-uuid-string.log.1731935033496.gz",
144+
wantUUID: "",
145+
wantError: true,
146+
},
147+
{
148+
name: "UUID at the end of filename",
149+
filename: "app.log.1731935033496.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.gz",
150+
wantUUID: "8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d",
151+
wantError: false,
152+
},
153+
}
154+
155+
for _, tt := range tests {
156+
tt := tt // create a new variable to hold the value of tt to avoid being overwritten by the next iteration (needed until Go 1.23)
157+
t.Run(tt.name, func(t *testing.T) {
158+
t.Parallel()
159+
gotUUID, err := GetUUIDFromFileName(tt.filename)
160+
if tt.wantError {
161+
g.Expect(err).To(gomega.HaveOccurred())
162+
} else {
163+
g.Expect(err).NotTo(gomega.HaveOccurred())
164+
g.Expect(gotUUID).To(gomega.Equal(tt.wantUUID))
165+
}
166+
})
167+
}
168+
}
169+
170+
func FuzzGetUUIDFromFileName(f *testing.F) {
171+
testcases := []string{
172+
"app.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.log.1731935033496.gz",
173+
"app.123e4567-e89b-12d3-a456-426614174000.log.1731935033496.gz",
174+
"dev.log.1731491904032.gz",
175+
"app.invalid-uuid-string.log.1731935033496.gz",
176+
"app.log.1731935033496.8ce1cc69-e1bb-4fe3-9613-e3eb1c5f5c4d.gz",
177+
}
178+
179+
for _, tc := range testcases {
180+
f.Add(tc)
181+
}
182+
183+
f.Fuzz(func(t *testing.T, filename string) {
184+
_, _ = GetUUIDFromFileName(filename)
185+
})
186+
}

0 commit comments

Comments
 (0)