Skip to content

Commit 479db6e

Browse files
committed
feat(atlas): add skip_patterns ignore globs and --stdin path list
skip_patterns compiles to a globset matcher applied to the index and update repo walks, so ignored files never enter the index. Config keys are now accepted flat as well as under [atlas], with [atlas] winning. atlas index --stdin reads paths from stdin (plain or git porcelain) instead of walking the repo, indexing only what it is given. Description parsing no longer hard-fails when a model omits the SHORT:/LONG: markers; the first line becomes the short description.
1 parent 8468fba commit 479db6e

11 files changed

Lines changed: 333 additions & 20 deletions

File tree

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ tree-sitter-typescript = "0.23"
5858
tree-sitter-python = "0.23"
5959
tree-sitter-go = "0.23"
6060
async-channel = "2.3"
61+
globset = "0.4"
6162
ignore = "0.4"
6263
rusqlite = { version = "0.34", features = ["bundled"] }
6364
sha2 = "0.10"

crates/core/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ md5 = "0.7"
2626
sha2 = { workspace = true }
2727
thiserror = { workspace = true }
2828
toml = { workspace = true }
29+
globset = { workspace = true }
2930
tree-sitter = { workspace = true }
3031

3132
[dev-dependencies]

crates/core/src/atlas/config.rs

Lines changed: 88 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use std::fmt;
33
use std::path::{Path, PathBuf};
44
use std::str::FromStr;
55

6+
use globset::{Glob, GlobSet, GlobSetBuilder};
67
use serde::Deserialize;
78

89
// ---------------------------------------------------------------------------
@@ -175,8 +176,22 @@ fn default_directory_llm() -> LlmProviderConfig {
175176
// Raw serde target (private)
176177
// ---------------------------------------------------------------------------
177178

179+
/// Top-level config file that supports both flat and `[atlas]`-nested formats.
180+
///
181+
/// Flat: `skip_patterns = [...]`
182+
/// Nested: `[atlas]\nskip_patterns = [...]`
183+
///
184+
/// Nested `[atlas]` fields take precedence over flat fields when both are present.
178185
#[derive(Debug, Deserialize, Default)]
179186
#[serde(default)]
187+
struct RawConfigFile {
188+
#[serde(flatten)]
189+
flat: RawConfig,
190+
atlas: Option<RawConfig>,
191+
}
192+
193+
#[derive(Debug, Deserialize, Default, Clone)]
194+
#[serde(default)]
180195
struct RawConfig {
181196
primer_path: Option<String>,
182197
db_path: Option<String>,
@@ -186,7 +201,7 @@ struct RawConfig {
186201
directory_llm: Option<RawLlmProvider>,
187202
}
188203

189-
#[derive(Debug, Deserialize, Default)]
204+
#[derive(Debug, Deserialize, Default, Clone)]
190205
#[serde(default)]
191206
struct RawLlmProvider {
192207
kind: Option<String>,
@@ -207,7 +222,10 @@ pub fn parse_config(
207222
env_vars: &HashMap<String, String>,
208223
) -> Result<AtlasConfig, ConfigError> {
209224
let raw: RawConfig = match toml_content {
210-
Some(content) => toml::from_str(content)?,
225+
Some(content) => {
226+
let file: RawConfigFile = toml::from_str(content)?;
227+
merge_raw_config(file.flat, file.atlas)
228+
}
211229
None => RawConfig::default(),
212230
};
213231

@@ -277,6 +295,45 @@ pub fn parse_config(
277295
})
278296
}
279297

298+
/// Merge flat (top-level) and nested (`[atlas]`) config. Nested values take precedence.
299+
fn merge_raw_config(flat: RawConfig, nested: Option<RawConfig>) -> RawConfig {
300+
let Some(nested) = nested else {
301+
return flat;
302+
};
303+
RawConfig {
304+
primer_path: nested.primer_path.or(flat.primer_path),
305+
db_path: nested.db_path.or(flat.db_path),
306+
max_file_tokens: nested.max_file_tokens.or(flat.max_file_tokens),
307+
skip_patterns: nested.skip_patterns.or(flat.skip_patterns),
308+
file_llm: nested.file_llm.or(flat.file_llm),
309+
directory_llm: nested.directory_llm.or(flat.directory_llm),
310+
}
311+
}
312+
313+
/// Compiled set of ignore patterns. Wraps `globset::GlobSet` to avoid
314+
/// leaking the third-party type into the public API.
315+
#[derive(Debug, Clone)]
316+
pub struct IgnoreMatcher(GlobSet);
317+
318+
impl IgnoreMatcher {
319+
/// Test whether a path matches any ignore pattern.
320+
pub fn is_match(&self, path: &std::path::Path) -> bool {
321+
self.0.is_match(path)
322+
}
323+
}
324+
325+
/// Compile skip patterns into an [`IgnoreMatcher`].
326+
///
327+
/// Each pattern is a glob expression (e.g. `"*.log"`, `"vendor/**"`).
328+
/// An empty slice produces a matcher that matches nothing.
329+
pub fn build_ignore_matcher(patterns: &[String]) -> Result<IgnoreMatcher, globset::Error> {
330+
let mut builder = GlobSetBuilder::new();
331+
for pattern in patterns {
332+
builder.add(Glob::new(pattern)?);
333+
}
334+
builder.build().map(IgnoreMatcher)
335+
}
336+
280337
/// Treat empty-or-whitespace strings as `None` so they fall through to defaults.
281338
fn non_empty(s: &str) -> Option<&str> {
282339
let trimmed = s.trim();
@@ -630,4 +687,33 @@ mod tests {
630687
"http://custom:9999"
631688
);
632689
}
690+
691+
// -- build_ignore_matcher --
692+
693+
#[test]
694+
fn ignore_matcher_extension_patterns() {
695+
let patterns = vec!["*.md".to_string(), "*.json".to_string()];
696+
let matcher = build_ignore_matcher(&patterns).unwrap();
697+
assert!(matcher.is_match(Path::new("README.md")));
698+
assert!(matcher.is_match(Path::new("src/config.json")));
699+
assert!(!matcher.is_match(Path::new("src/main.rs")));
700+
assert!(!matcher.is_match(Path::new("lib.py")));
701+
}
702+
703+
#[test]
704+
fn ignore_matcher_directory_glob() {
705+
let patterns = vec!["packages/mom/**".to_string()];
706+
let matcher = build_ignore_matcher(&patterns).unwrap();
707+
assert!(matcher.is_match(Path::new("packages/mom/index.js")));
708+
assert!(matcher.is_match(Path::new("packages/mom/src/lib.rs")));
709+
assert!(!matcher.is_match(Path::new("packages/dad/index.js")));
710+
assert!(!matcher.is_match(Path::new("src/main.rs")));
711+
}
712+
713+
#[test]
714+
fn ignore_matcher_empty_patterns_match_nothing() {
715+
let matcher = build_ignore_matcher(&[]).unwrap();
716+
assert!(!matcher.is_match(Path::new("anything.rs")));
717+
assert!(!matcher.is_match(Path::new("some/path/file.txt")));
718+
}
633719
}

crates/core/src/atlas/mod.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ pub mod types;
99

1010
pub use changes::{affected_directories, compute_change_set, ChangeSet};
1111
pub use config::{
12-
parse_config, AtlasConfig, BaseUrl, ConfigError, DbPath, LlmProviderConfig, LlmProviderKind,
13-
ModelName, PrimerPath,
12+
build_ignore_matcher, parse_config, AtlasConfig, BaseUrl, ConfigError, DbPath, IgnoreMatcher,
13+
LlmProviderConfig, LlmProviderKind, ModelName, PrimerPath,
1414
};
1515
pub use hash::content_hash;
16-
pub use parse::{parse_description, FileDescription, ParseDescriptionError};
16+
pub use parse::{parse_description, parse_stdin_line, FileDescription, ParseDescriptionError};
1717
pub use prompts::{
1818
build_directory_prompt, build_file_prompt, build_primer_refinement_prompt,
1919
directory_system_prompt, estimate_tokens, file_system_prompt, truncate_to_tokens,

crates/core/src/atlas/parse.rs

Lines changed: 165 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,13 @@ pub fn parse_description(response: &str) -> Result<FileDescription, ParseDescrip
3131
let lines: Vec<&str> = response.lines().collect();
3232

3333
// Find first line starting with "SHORT:"
34-
let short_idx = lines
34+
let short_idx = match lines
3535
.iter()
3636
.position(|l| l.trim_start().starts_with("SHORT:"))
37-
.ok_or(ParseDescriptionError::MissingShort)?;
37+
{
38+
Some(idx) => idx,
39+
None => return parse_description_fallback(response),
40+
};
3841

3942
let short = lines[short_idx]
4043
.trim_start()
@@ -78,6 +81,68 @@ pub fn parse_description(response: &str) -> Result<FileDescription, ParseDescrip
7881
Ok(FileDescription { short, long })
7982
}
8083

84+
/// Fallback parser for LLM responses that lack SHORT:/LONG: markers.
85+
///
86+
/// Uses the first non-empty line (truncated to 80 chars) as the short
87+
/// description and the remaining text as the long description.
88+
fn parse_description_fallback(response: &str) -> Result<FileDescription, ParseDescriptionError> {
89+
let trimmed = response.trim();
90+
if trimmed.is_empty() {
91+
return Err(ParseDescriptionError::MissingShort);
92+
}
93+
94+
let mut lines = trimmed.lines();
95+
let first_line = lines
96+
.next()
97+
.map(|l| l.trim())
98+
.filter(|l| !l.is_empty())
99+
.ok_or(ParseDescriptionError::EmptyShort)?;
100+
101+
let short: String = first_line.chars().take(80).collect();
102+
103+
let long: String = lines.collect::<Vec<_>>().join("\n").trim().to_string();
104+
105+
if long.is_empty() {
106+
// Use the short as long too — better than failing
107+
return Ok(FileDescription {
108+
short: short.clone(),
109+
long: short,
110+
});
111+
}
112+
113+
Ok(FileDescription { short, long })
114+
}
115+
116+
/// Parse a single line from stdin into a file path, or `None` to skip.
117+
///
118+
/// Handles:
119+
/// - Plain paths (`src/foo.ts`)
120+
/// - Git porcelain format (`M src/foo.ts`, `?? new_file.ts`)
121+
/// - Renames (`R old.ts -> new.ts` — returns the new path)
122+
/// - Returns `None` for empty lines and `#` comments
123+
pub fn parse_stdin_line(line: &str) -> Option<&str> {
124+
let trimmed = line.trim();
125+
if trimmed.is_empty() || trimmed.starts_with('#') {
126+
return None;
127+
}
128+
129+
// Detect git status --porcelain format: "XY path" where X/Y are status chars
130+
// and position 2 is a space. Status chars: A, M, D, R, C, U, ?, !
131+
let is_porcelain = trimmed.len() > 3
132+
&& trimmed.as_bytes()[2] == b' '
133+
&& (trimmed.as_bytes()[0].is_ascii_alphabetic()
134+
|| trimmed.as_bytes()[0] == b'?'
135+
|| trimmed.as_bytes()[0] == b'!');
136+
137+
if is_porcelain {
138+
let rest = &trimmed[3..];
139+
// Handle renames: "old -> new", take the new path
140+
Some(rest.split(" -> ").last().unwrap_or(rest))
141+
} else {
142+
Some(trimmed)
143+
}
144+
}
145+
81146
#[cfg(test)]
82147
mod tests {
83148
use super::*;
@@ -123,10 +188,11 @@ and starts the server.";
123188
}
124189

125190
#[test]
126-
fn missing_short_returns_error() {
191+
fn missing_short_uses_fallback() {
192+
// Without SHORT: prefix, fallback parser uses first line as short
127193
let input = "LONG: Some long description";
128-
let err = parse_description(input).unwrap_err();
129-
assert!(matches!(err, ParseDescriptionError::MissingShort));
194+
let desc = parse_description(input).unwrap();
195+
assert_eq!(desc.short, "LONG: Some long description");
130196
}
131197

132198
#[test]
@@ -167,16 +233,106 @@ and starts the server.";
167233
}
168234

169235
#[test]
170-
fn case_sensitive_short_required() {
236+
fn case_sensitive_short_falls_back() {
237+
// Without "SHORT:" prefix, fallback parser kicks in
171238
let input = "short: lowercase\nLONG: Something";
172-
let err = parse_description(input).unwrap_err();
173-
assert!(matches!(err, ParseDescriptionError::MissingShort));
239+
let desc = parse_description(input).unwrap();
240+
assert_eq!(desc.short, "short: lowercase");
241+
assert_eq!(desc.long, "LONG: Something");
174242
}
175243

176244
#[test]
177-
fn case_sensitive_long_required() {
245+
fn case_sensitive_long_falls_back_from_main_to_use_remaining() {
178246
let input = "SHORT: Valid\nlong: lowercase";
247+
// SHORT: found, but no LONG: — falls back to treating remaining lines as long
179248
let err = parse_description(input).unwrap_err();
180249
assert!(matches!(err, ParseDescriptionError::MissingLong));
181250
}
251+
252+
// -- Fallback parser tests --
253+
254+
#[test]
255+
fn fallback_freeform_response_uses_first_line_as_short() {
256+
let input = "This file handles authentication.\nIt validates tokens and manages sessions.";
257+
let desc = parse_description(input).unwrap();
258+
assert_eq!(desc.short, "This file handles authentication.");
259+
assert_eq!(desc.long, "It validates tokens and manages sessions.");
260+
}
261+
262+
#[test]
263+
fn fallback_single_line_uses_same_for_both() {
264+
let input = "A utility module for string processing.";
265+
let desc = parse_description(input).unwrap();
266+
assert_eq!(desc.short, "A utility module for string processing.");
267+
assert_eq!(desc.long, "A utility module for string processing.");
268+
}
269+
270+
#[test]
271+
fn fallback_truncates_short_to_80_chars() {
272+
let input = "a".repeat(120) + "\nSome long description here.";
273+
let desc = parse_description(&input).unwrap();
274+
assert_eq!(desc.short.len(), 80);
275+
assert_eq!(desc.long, "Some long description here.");
276+
}
277+
278+
#[test]
279+
fn fallback_empty_response_fails() {
280+
let input = "";
281+
let err = parse_description(input).unwrap_err();
282+
assert!(matches!(err, ParseDescriptionError::MissingShort));
283+
}
284+
285+
// -- parse_stdin_line tests --
286+
287+
#[test]
288+
fn stdin_line_plain_path() {
289+
assert_eq!(parse_stdin_line("src/foo.ts"), Some("src/foo.ts"));
290+
}
291+
292+
#[test]
293+
fn stdin_line_porcelain_modified() {
294+
assert_eq!(parse_stdin_line("M src/foo.ts"), Some("src/foo.ts"));
295+
}
296+
297+
#[test]
298+
fn stdin_line_porcelain_added() {
299+
assert_eq!(parse_stdin_line("A src/new.ts"), Some("src/new.ts"));
300+
}
301+
302+
#[test]
303+
fn stdin_line_porcelain_untracked() {
304+
assert_eq!(parse_stdin_line("?? src/new.ts"), Some("src/new.ts"));
305+
}
306+
307+
#[test]
308+
fn stdin_line_porcelain_rename() {
309+
assert_eq!(parse_stdin_line("R old.ts -> new.ts"), Some("new.ts"));
310+
}
311+
312+
#[test]
313+
fn stdin_line_empty() {
314+
assert_eq!(parse_stdin_line(""), None);
315+
}
316+
317+
#[test]
318+
fn stdin_line_whitespace_only() {
319+
assert_eq!(parse_stdin_line(" "), None);
320+
}
321+
322+
#[test]
323+
fn stdin_line_comment() {
324+
assert_eq!(parse_stdin_line("# this is a comment"), None);
325+
}
326+
327+
#[test]
328+
fn stdin_line_short_path_not_porcelain() {
329+
// "ab" is only 2 chars — too short to be porcelain format
330+
assert_eq!(parse_stdin_line("ab"), Some("ab"));
331+
}
332+
333+
#[test]
334+
fn stdin_line_path_starting_with_question_mark_not_porcelain() {
335+
// "?readme.txt" has no space at position 2, so it's a plain path
336+
assert_eq!(parse_stdin_line("?readme.txt"), Some("?readme.txt"));
337+
}
182338
}

0 commit comments

Comments
 (0)