Skip to content

Commit 6a33ebb

Browse files
authored
Optimize glob walkdir (#3013)
* feat(storage): optimize glob func * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * feat(rgafs): implement paged glob traversal without full tree materialization * fix(localfs): offload blocking fs operations to spawn_blocking * feat(glob): cap glob api default node_limit at 256 * feat(sdk): add node_limit options for glob in python and go SDKs
1 parent 8d861fa commit 6a33ebb

44 files changed

Lines changed: 2069 additions & 165 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

benchmark/custom/session_contention_benchmark.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -994,7 +994,7 @@ async def _run_retrieval_operation(
994994
"glob",
995995
lambda: adapter.glob(
996996
uri=self.config.data_root_uri,
997-
pattern="*.md",
997+
pattern="**/*.md",
998998
limit=self.config.find_limit,
999999
),
10001000
worker_id=worker_id,

crates/ov_cli/test_ov.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ fi
203203
echo ""
204204

205205
echo "5.4. Glob pattern search..."
206-
if $OV_BIN glob "*.md" --uri "viking://resources"; then
206+
if $OV_BIN glob "**/*.md" --uri "viking://resources"; then
207207
print_success "Glob search completed"
208208
else
209209
print_error "Glob search failed"

crates/ragfs-python/src/lib.rs

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use ragfs::cache::{
2020
use ragfs::core::builder::EncryptionConfig;
2121
use ragfs::core::{
2222
build_default_stack, register_builtin_plugins, ConfigValue, FileInfo, FileSystem,
23-
FilesystemStats, FsContext, FsContextInner, FsOperation, GrepResult, MountableFS,
23+
FilesystemStats, FsContext, FsContextInner, FsOperation, GlobPage, GrepResult, MountableFS,
2424
OperationStats, PluginConfig, RagfsConfig, StatsWrappedFS, TreeEntry, WriteFlag, FS_CTX,
2525
};
2626

@@ -1618,6 +1618,57 @@ impl RAGFSBindingClient {
16181618
})
16191619
}
16201620

1621+
/// Return one page of flat glob results.
1622+
///
1623+
/// Args:
1624+
/// path: The root path of the traversal
1625+
/// pattern: Glob pattern matched against query-root-relative paths
1626+
/// show_hidden: Whether to include hidden files (default: False)
1627+
/// page_size: Maximum number of matched entries returned in this page
1628+
/// level_limit: Maximum depth relative to query root (default: None)
1629+
/// continuation_token: Opaque token returned by the previous page
1630+
/// ctx: Optional FsContext dict (e.g. {"account_id": ...})
1631+
///
1632+
/// Returns:
1633+
/// A dict with keys: entries (list[GlobEntry]), next_token (str | None)
1634+
#[pyo3(signature = (path, pattern, show_hidden=false, page_size=None, level_limit=None, continuation_token=None, ctx=None))]
1635+
fn glob_directory(
1636+
&self,
1637+
py: Python<'_>,
1638+
path: String,
1639+
pattern: String,
1640+
show_hidden: bool,
1641+
page_size: Option<i32>,
1642+
level_limit: Option<i32>,
1643+
continuation_token: Option<String>,
1644+
ctx: Option<HashMap<String, String>>,
1645+
) -> PyResult<Py<PyAny>> {
1646+
let fs_ctx = build_fs_context(ctx);
1647+
let top = self.top.clone();
1648+
let page_size = page_size.map(|n| if n < 0 { 0 } else { n as usize });
1649+
let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
1650+
1651+
let page: GlobPage = self
1652+
.run_scoped(py, fs_ctx, move || async move {
1653+
top.glob_directory(
1654+
&path,
1655+
&pattern,
1656+
show_hidden,
1657+
page_size,
1658+
level_limit_usize,
1659+
continuation_token,
1660+
)
1661+
.await
1662+
})
1663+
.map_err(to_py_err)?;
1664+
1665+
Python::attach(|py| {
1666+
let value = serde_json::to_value(&page)
1667+
.map_err(|err| PyRuntimeError::new_err(err.to_string()))?;
1668+
serde_json_to_py(py, &value)
1669+
})
1670+
}
1671+
16211672
/// Query multi-write sync status under a file or directory path.
16221673
///
16231674
/// Args:

crates/ragfs/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ lru = "0.12"
8080
# Regular expressions for grep
8181
regex = "1.10"
8282
mime_guess = "2.0"
83+
globset = "0.4"
8384

8485
# Encryption (envelope encryption: AES-256-GCM + HKDF-SHA256)
8586
aes-gcm = "0.10"

crates/ragfs/src/cache/wrapper.rs

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ use crate::core::filesystem::{
99
relative_match_file,
1010
};
1111
use crate::core::{
12-
FileInfo, FileSystem, GrepMatch, GrepResult, MultiWriteWrappedFS, Result, TreeEntry, WriteFlag,
12+
FileInfo, FileSystem, GlobPage, GrepMatch, GrepResult, MultiWriteWrappedFS, Result,
13+
TreeEntry, WriteFlag,
1314
};
1415
use async_trait::async_trait;
1516
use bytes::Bytes;
@@ -1233,6 +1234,27 @@ impl FileSystem for CachedFileSystem {
12331234
.tree_directory(path, show_hidden, node_limit, level_limit)
12341235
.await
12351236
}
1237+
1238+
async fn glob_directory(
1239+
&self,
1240+
path: &str,
1241+
pattern: &str,
1242+
show_hidden: bool,
1243+
page_size: Option<usize>,
1244+
level_limit: Option<usize>,
1245+
continuation_token: Option<String>,
1246+
) -> Result<GlobPage> {
1247+
self.backend
1248+
.glob_directory(
1249+
path,
1250+
pattern,
1251+
show_hidden,
1252+
page_size,
1253+
level_limit,
1254+
continuation_token,
1255+
)
1256+
.await
1257+
}
12361258
}
12371259

12381260
fn normalize_path(path: &str) -> String {

crates/ragfs/src/core/encryption_wrapper.rs

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use crate::shape::SHAPE_MANIFEST_PATH;
2222
use super::context::FsContextView;
2323
use super::errors::{Error, Result};
2424
use super::filesystem::{compile_grep_regex, normalize_prefix_path, FileSystem};
25-
use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};
25+
use super::types::{FileInfo, GlobPage, GrepResult, TreeEntry, WriteFlag};
2626

2727
const SYSTEM_ACCOUNT_ID: &str = "_system";
2828
const TEMP_ROOT_CACHE_TTL: Duration = Duration::from_secs(15 * 60);
@@ -438,6 +438,31 @@ impl FileSystem for EncryptionWrappedFS {
438438
.collect())
439439
}
440440

441+
async fn glob_directory(
442+
&self,
443+
path: &str,
444+
pattern: &str,
445+
show_hidden: bool,
446+
page_size: Option<usize>,
447+
level_limit: Option<usize>,
448+
continuation_token: Option<String>,
449+
) -> Result<GlobPage> {
450+
let mut page = self
451+
.inner
452+
.glob_directory(
453+
path,
454+
pattern,
455+
show_hidden,
456+
page_size,
457+
level_limit,
458+
continuation_token,
459+
)
460+
.await?;
461+
page.entries
462+
.retain(|entry| !Self::is_shape_manifest_path(&entry.path));
463+
Ok(page)
464+
}
465+
441466
async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
442467
self.inner.ensure_parent_dirs(path, mode).await
443468
}

crates/ragfs/src/core/filesystem.rs

Lines changed: 198 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ use std::any::Any;
1010
use std::cmp::Ordering;
1111

1212
use super::errors::{Error, Result};
13-
use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};
13+
use super::glob::{
14+
compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,
15+
};
16+
use super::types::{FileInfo, GlobEntry, GlobPage, GrepResult, TreeEntry, WriteFlag};
1417

1518
/// Normalize a path for prefix comparisons.
1619
///
@@ -470,6 +473,65 @@ pub trait FileSystem: Send + Sync + Any {
470473
Ok(result)
471474
}
472475

476+
/// Return one page of flat glob results under `path`.
477+
///
478+
/// The default implementation preserves the current Python behavior by
479+
/// reusing `tree_directory()` and matching against the returned `rel_path`
480+
/// values, then slicing matches with an opaque continuation token.
481+
async fn glob_directory(
482+
&self,
483+
path: &str,
484+
pattern: &str,
485+
show_hidden: bool,
486+
page_size: Option<usize>,
487+
level_limit: Option<usize>,
488+
continuation_token: Option<String>,
489+
) -> Result<GlobPage> {
490+
let matcher = PreparedGlob::new(pattern)?;
491+
if matches!(page_size, Some(0)) {
492+
return Err(Error::invalid_operation("page_size must be positive"));
493+
}
494+
495+
let entries = self
496+
.tree_directory(path, show_hidden, None, level_limit)
497+
.await?;
498+
499+
let mut matched = Vec::new();
500+
for entry in entries {
501+
if matcher.is_match(&entry.rel_path) {
502+
matched.push(GlobEntry {
503+
path: entry.path,
504+
rel_path: entry.rel_path,
505+
name: entry.info.name,
506+
is_dir: entry.info.is_dir,
507+
});
508+
}
509+
}
510+
matched.sort_by(|left, right| compare_rel_paths(&left.rel_path, &right.rel_path));
511+
512+
let start = decode_offset_token(
513+
continuation_token.as_deref(),
514+
path,
515+
pattern,
516+
show_hidden,
517+
level_limit,
518+
)?;
519+
if start > matched.len() {
520+
return Err(Error::invalid_operation("continuation token out of range"));
521+
}
522+
let end = page_size
523+
.map(|limit| start.saturating_add(limit))
524+
.unwrap_or(matched.len())
525+
.min(matched.len());
526+
let next_token = (end < matched.len())
527+
.then(|| encode_offset_token(end, path, pattern, show_hidden, level_limit));
528+
529+
Ok(GlobPage {
530+
entries: matched[start..end].to_vec(),
531+
next_token,
532+
})
533+
}
534+
473535
/// Internal recursive helper for tree_directory.
474536
///
475537
/// # Arguments
@@ -1063,4 +1125,139 @@ mod tests {
10631125
assert!(names.contains(&"secret.txt".to_string()));
10641126
assert!(!names.contains(&".hidden_file".to_string()));
10651127
}
1128+
1129+
/// Test helper that calls `glob_directory` with a fixed `/root` query root.
1130+
///
1131+
/// Args:
1132+
/// - `fs`: The `TreeFS` instance under test.
1133+
/// - `pattern`: The glob pattern to match.
1134+
/// - `page_size`: The requested page size.
1135+
/// - `continuation_token`: The pagination token for the next page.
1136+
///
1137+
/// Returns:
1138+
/// - A `GlobPage` on success. In tests this helper uses `unwrap()`, so any
1139+
/// error fails the test immediately.
1140+
async fn root_glob(
1141+
fs: &TreeFS,
1142+
pattern: &str,
1143+
page_size: Option<usize>,
1144+
continuation_token: Option<String>,
1145+
) -> crate::core::GlobPage {
1146+
fs.glob_directory("/root", pattern, false, page_size, None, continuation_token)
1147+
.await
1148+
.unwrap()
1149+
}
1150+
1151+
/// Test helper that extracts each entry's `rel_path` from a `GlobPage`.
1152+
///
1153+
/// Args:
1154+
/// - `page`: The glob page whose relative paths should be collected.
1155+
///
1156+
/// Returns:
1157+
/// - A list of `rel_path` values in their original order, suitable for
1158+
/// result-content and ordering assertions.
1159+
fn glob_rel_paths(page: &crate::core::GlobPage) -> Vec<String> {
1160+
page.entries
1161+
.iter()
1162+
.map(|entry| entry.rel_path.clone())
1163+
.collect()
1164+
}
1165+
1166+
#[tokio::test]
1167+
async fn test_glob_directory_matches_full_relative_path_semantics() {
1168+
let fs = TreeFS::default()
1169+
.with_dir_entries("/root", vec![("sub", true), ("top.md", false)])
1170+
.with_dir_entries(
1171+
"/root/sub",
1172+
vec![("nested.md", false), ("nested.txt", false)],
1173+
);
1174+
1175+
let page = root_glob(&fs, "**/*.md", None, None).await;
1176+
1177+
assert_eq!(glob_rel_paths(&page), vec!["sub/nested.md", "top.md"]);
1178+
assert!(page.next_token.is_none());
1179+
}
1180+
1181+
#[tokio::test]
1182+
async fn test_glob_directory_anchors_multi_segment_patterns_at_root() {
1183+
let fs = TreeFS::default()
1184+
.with_dir_entries("/root", vec![("a", true), ("x", true)])
1185+
.with_dir_entries("/root/a", vec![("b", true)])
1186+
.with_dir_entries("/root/a/b", vec![("c.md", false)])
1187+
.with_dir_entries("/root/x", vec![("a", true)])
1188+
.with_dir_entries("/root/x/a", vec![("b", true)])
1189+
.with_dir_entries("/root/x/a/b", vec![("c.md", false)]);
1190+
1191+
let page = root_glob(&fs, "a/**/*.md", None, None).await;
1192+
1193+
assert_eq!(glob_rel_paths(&page), vec!["a/b/c.md"]);
1194+
}
1195+
1196+
#[tokio::test]
1197+
async fn test_glob_directory_paginates_with_opaque_offset_tokens() {
1198+
let fs = TreeFS::default().with_dir_entries(
1199+
"/root",
1200+
vec![("a.md", false), ("b.md", false), ("c.md", false)],
1201+
);
1202+
1203+
let first = root_glob(&fs, "*.md", Some(2), None).await;
1204+
assert_eq!(glob_rel_paths(&first), vec!["a.md", "b.md"]);
1205+
assert!(first.next_token.is_some());
1206+
1207+
let second = root_glob(&fs, "*.md", Some(2), first.next_token).await;
1208+
assert_eq!(glob_rel_paths(&second), vec!["c.md"]);
1209+
assert!(second.next_token.is_none());
1210+
}
1211+
1212+
#[tokio::test]
1213+
async fn test_glob_directory_rejects_token_from_different_query_scope() {
1214+
let fs = TreeFS::default().with_dir_entries(
1215+
"/root",
1216+
vec![("a.md", false), ("b.md", false), ("c.md", false)],
1217+
);
1218+
1219+
let first = root_glob(&fs, "*.md", Some(2), None).await;
1220+
let err = fs
1221+
.glob_directory("/root", "*.txt", false, Some(2), None, first.next_token)
1222+
.await
1223+
.unwrap_err();
1224+
1225+
assert!(matches!(err, Error::InvalidOperation(_)));
1226+
}
1227+
1228+
#[tokio::test]
1229+
async fn test_glob_directory_empty_pattern_is_invalid() {
1230+
let fs = TreeFS::default().with_dir_entries("/root", vec![("a.md", false)]);
1231+
1232+
let err = fs
1233+
.glob_directory("/root", "", false, None, None, None)
1234+
.await
1235+
.unwrap_err();
1236+
1237+
assert!(matches!(err, Error::InvalidOperation(_)));
1238+
}
1239+
1240+
#[tokio::test]
1241+
async fn test_glob_directory_empty_pattern_is_invalid_for_empty_directory() {
1242+
let fs = TreeFS::default().with_dir_entries("/root", vec![]);
1243+
1244+
let err = fs
1245+
.glob_directory("/root", "", false, None, None, None)
1246+
.await
1247+
.unwrap_err();
1248+
1249+
assert!(matches!(err, Error::InvalidOperation(_)));
1250+
}
1251+
1252+
#[tokio::test]
1253+
async fn test_glob_directory_zero_page_size_is_invalid() {
1254+
let fs = TreeFS::default().with_dir_entries("/root", vec![("a.md", false)]);
1255+
1256+
let err = fs
1257+
.glob_directory("/root", "*.md", false, Some(0), None, None)
1258+
.await
1259+
.unwrap_err();
1260+
1261+
assert!(matches!(err, Error::InvalidOperation(_)));
1262+
}
10661263
}

0 commit comments

Comments
 (0)