Skip to content

Commit 5c29e24

Browse files
sergeyevstifeevzachvictor
sergeyevstifeev
authored and
zachvictor
committed
Implement matchGlob support in list queries
1 parent 7503535 commit 5c29e24

2 files changed

Lines changed: 237 additions & 0 deletions

File tree

‎fakestorage/object.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"fmt"
1515
"io"
1616
"net/http"
17+
"regexp"
1718
"slices"
1819
"sort"
1920
"strconv"
@@ -347,6 +348,7 @@ func (s *Server) triggerEvent(obj *backend.StreamingObject, eventType notificati
347348

348349
type ListOptions struct {
349350
Prefix string
351+
MatchGlob string
350352
Delimiter string
351353
Versions bool
352354
StartOffset string
@@ -402,6 +404,11 @@ func (s *Server) ListObjectsWithOptionsPaginated(bucketName string, options List
402404
if !strings.HasPrefix(obj.Name, options.Prefix) {
403405
continue
404406
}
407+
408+
if options.MatchGlob != "" && !matchGlob(options.MatchGlob, obj.Name) {
409+
continue
410+
}
411+
405412
objName := strings.Replace(obj.Name, options.Prefix, "", 1)
406413
delimPos := strings.Index(objName, options.Delimiter)
407414
if options.Delimiter != "" && delimPos > -1 {
@@ -678,6 +685,7 @@ func (s *Server) listObjects(r *http.Request) jsonResponse {
678685
}
679686
response, err := s.ListObjectsWithOptionsPaginated(bucketName, ListOptions{
680687
Prefix: r.URL.Query().Get("prefix"),
688+
MatchGlob: r.URL.Query().Get("matchGlob"),
681689
Delimiter: r.URL.Query().Get("delimiter"),
682690
Versions: r.URL.Query().Get("versions") == "true",
683691
StartOffset: r.URL.Query().Get("startOffset"),
@@ -697,6 +705,7 @@ func (s *Server) xmlListObjects(r *http.Request) xmlResponse {
697705

698706
opts := ListOptions{
699707
Prefix: r.URL.Query().Get("prefix"),
708+
MatchGlob: r.URL.Query().Get("matchGlob"),
700709
Delimiter: r.URL.Query().Get("delimiter"),
701710
Versions: r.URL.Query().Get("versions") == "true",
702711
}
@@ -1453,3 +1462,120 @@ func (s *Server) composeObject(r *http.Request) jsonResponse {
14531462

14541463
return jsonResponse{data: newObjectResponse(obj.ObjectAttrs, urlhelper.GetBaseURL(r))}
14551464
}
1465+
1466+
// matchGlob matches a glob pattern against a string following GCS glob syntax.
1467+
// See https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
1468+
// Supports:
1469+
// - * matches zero or more characters (excluding /)
1470+
// - ** matches zero or more characters (including /)
1471+
// - ? matches any single character (excluding /)
1472+
// - [abc] matches exactly one character in the set
1473+
// - [a-z] matches one character in a range
1474+
// - [!abc] or [^abc] matches one character NOT in the set
1475+
// - {abc,xyz} matches one of the specified options
1476+
// Doesn't currently support:
1477+
// - nested {} braces
1478+
// - patterns inside {} braces e.g. {*foo,bar*}
1479+
func matchGlob(pattern, name string) bool {
1480+
regex := globToRegex(pattern)
1481+
matched, err := regexp.MatchString(regex, name)
1482+
if err != nil {
1483+
return false
1484+
}
1485+
return matched
1486+
}
1487+
1488+
func globToRegex(pattern string) string {
1489+
var result strings.Builder
1490+
result.WriteString("^")
1491+
1492+
i := 0
1493+
for i < len(pattern) {
1494+
switch pattern[i] {
1495+
case '*':
1496+
if i+1 < len(pattern) && pattern[i+1] == '*' {
1497+
// ** matches zero or more characters including /
1498+
result.WriteString(".*")
1499+
i += 2
1500+
} else {
1501+
// * matches zero or more characters excluding /
1502+
result.WriteString("[^/]*")
1503+
i++
1504+
}
1505+
case '?':
1506+
// ? matches any single character excluding /
1507+
result.WriteString("[^/]")
1508+
i++
1509+
case '[':
1510+
// Character class [abc] or [a-z] or [!abc] or [^abc]
1511+
j := i + 1
1512+
if j < len(pattern) && (pattern[j] == '!' || pattern[j] == '^') {
1513+
j++
1514+
}
1515+
for j < len(pattern) && pattern[j] != ']' {
1516+
j++
1517+
}
1518+
if j < len(pattern) {
1519+
// Valid character class
1520+
charClass := pattern[i : j+1]
1521+
// Convert [!...] to [^...] for regex
1522+
if strings.HasPrefix(charClass, "[!") {
1523+
charClass = "[^" + charClass[2:]
1524+
}
1525+
result.WriteString(charClass)
1526+
i = j + 1
1527+
} else {
1528+
// Invalid character class, treat as literal
1529+
result.WriteString(regexp.QuoteMeta(string(pattern[i])))
1530+
i++
1531+
}
1532+
case '{':
1533+
// Brace expansion {abc,xyz}
1534+
j := i + 1
1535+
depth := 1
1536+
for j < len(pattern) && depth > 0 {
1537+
switch pattern[j] {
1538+
case '{':
1539+
depth++
1540+
case '}':
1541+
depth--
1542+
}
1543+
j++
1544+
}
1545+
if depth == 0 {
1546+
// Valid brace expansion
1547+
braceContent := pattern[i+1 : j-1]
1548+
options := strings.Split(braceContent, ",")
1549+
result.WriteString("(")
1550+
for k, option := range options {
1551+
if k > 0 {
1552+
result.WriteString("|")
1553+
}
1554+
result.WriteString(regexp.QuoteMeta(option))
1555+
}
1556+
result.WriteString(")")
1557+
i = j
1558+
} else {
1559+
// Invalid brace expansion, treat as literal
1560+
result.WriteString(regexp.QuoteMeta(string(pattern[i])))
1561+
i++
1562+
}
1563+
case '\\':
1564+
// Escape character
1565+
if i+1 < len(pattern) {
1566+
result.WriteString(regexp.QuoteMeta(string(pattern[i+1])))
1567+
i += 2
1568+
} else {
1569+
result.WriteString(regexp.QuoteMeta(string(pattern[i])))
1570+
i++
1571+
}
1572+
default:
1573+
// Regular character, escape for regex
1574+
result.WriteString(regexp.QuoteMeta(string(pattern[i])))
1575+
i++
1576+
}
1577+
}
1578+
1579+
result.WriteString("$")
1580+
return result.String()
1581+
}

‎fakestorage/object_test.go‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -994,6 +994,117 @@ func getTestCasesForListTests(versioningEnabled, withOverwrites bool) []listTest
994994
},
995995
nil,
996996
},
997+
{
998+
fmt.Sprintf("filtering MatchGlob *, versioning %t and overwrites %t", versioningEnabled, withOverwrites),
999+
"some-bucket",
1000+
&storage.Query{MatchGlob: "img/*.jpg"},
1001+
[]string{
1002+
"img/brand.jpg",
1003+
},
1004+
nil,
1005+
},
1006+
{
1007+
fmt.Sprintf("filtering MatchGlob **, versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1008+
"some-bucket",
1009+
&storage.Query{MatchGlob: "img/**.jpg"},
1010+
[]string{
1011+
"img/brand.jpg",
1012+
"img/hi-res/party-01.jpg",
1013+
"img/hi-res/party-02.jpg",
1014+
"img/hi-res/party-03.jpg",
1015+
"img/low-res/party-01.jpg",
1016+
"img/low-res/party-02.jpg",
1017+
"img/low-res/party-03.jpg",
1018+
},
1019+
nil,
1020+
},
1021+
{
1022+
fmt.Sprintf("filtering MatchGlob ** and *, versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1023+
"some-bucket",
1024+
&storage.Query{MatchGlob: "img/**/party*.jpg"},
1025+
[]string{
1026+
"img/hi-res/party-01.jpg",
1027+
"img/hi-res/party-02.jpg",
1028+
"img/hi-res/party-03.jpg",
1029+
"img/low-res/party-01.jpg",
1030+
"img/low-res/party-02.jpg",
1031+
"img/low-res/party-03.jpg",
1032+
},
1033+
nil,
1034+
},
1035+
{
1036+
fmt.Sprintf("filtering MatchGlob ** and [ab], versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1037+
"some-bucket",
1038+
&storage.Query{MatchGlob: "img/**/party-0[12].jpg"},
1039+
[]string{
1040+
"img/hi-res/party-01.jpg",
1041+
"img/hi-res/party-02.jpg",
1042+
"img/low-res/party-01.jpg",
1043+
"img/low-res/party-02.jpg",
1044+
},
1045+
nil,
1046+
},
1047+
{
1048+
fmt.Sprintf("filtering MatchGlob ** and [a-z], versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1049+
"some-bucket",
1050+
&storage.Query{MatchGlob: "img/**/party-0[2-3].jpg"},
1051+
[]string{
1052+
"img/hi-res/party-02.jpg",
1053+
"img/hi-res/party-03.jpg",
1054+
"img/low-res/party-02.jpg",
1055+
"img/low-res/party-03.jpg",
1056+
},
1057+
nil,
1058+
},
1059+
{
1060+
fmt.Sprintf("filtering MatchGlob ** and [!abc], versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1061+
"some-bucket",
1062+
&storage.Query{MatchGlob: "img/**/party-0[!13].jpg"},
1063+
[]string{
1064+
"img/hi-res/party-02.jpg",
1065+
"img/low-res/party-02.jpg",
1066+
},
1067+
nil,
1068+
},
1069+
{
1070+
fmt.Sprintf("filtering MatchGlob ** and {abc,xyz}, versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1071+
"some-bucket",
1072+
&storage.Query{MatchGlob: "img/**{brand,party-01,party-02}.jpg"},
1073+
[]string{
1074+
"img/brand.jpg",
1075+
"img/hi-res/party-01.jpg",
1076+
"img/hi-res/party-02.jpg",
1077+
"img/low-res/party-01.jpg",
1078+
"img/low-res/party-02.jpg",
1079+
},
1080+
nil,
1081+
},
1082+
{
1083+
fmt.Sprintf("filtering MatchGlob ** and [^abc], versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1084+
"some-bucket",
1085+
&storage.Query{MatchGlob: "img/**/party-0[^543].jpg"},
1086+
[]string{
1087+
"img/hi-res/party-01.jpg",
1088+
"img/hi-res/party-02.jpg",
1089+
"img/low-res/party-01.jpg",
1090+
"img/low-res/party-02.jpg",
1091+
},
1092+
nil,
1093+
},
1094+
{
1095+
fmt.Sprintf("filtering MatchGlob ** and \a, versioning %t and overwrites %t", versioningEnabled, withOverwrites),
1096+
"some-bucket",
1097+
&storage.Query{MatchGlob: "img/**/party\\-*.jpg"},
1098+
[]string{
1099+
"img/hi-res/party-01.jpg",
1100+
"img/hi-res/party-02.jpg",
1101+
"img/hi-res/party-03.jpg",
1102+
"img/low-res/party-01.jpg",
1103+
"img/low-res/party-02.jpg",
1104+
"img/low-res/party-03.jpg",
1105+
},
1106+
nil,
1107+
},
9971108
{
9981109
fmt.Sprintf("full prefix, versioning %t and overwrites %t", versioningEnabled, withOverwrites),
9991110
"some-bucket",

0 commit comments

Comments
 (0)