Extensively document and clean up css_lexer/css_parse (#78)

Yet another giant refactor:

- Removed `hdx_atom` crate, instead opting to do interning on each node.
This de-couples the lexer & parser from _css_ the language itself.
- Fold in `hdx_syntax` into `hdx_lexer` as it wasn't used outside of
that.
 - Rename `hdx_lexer` to `css_lexer`.
 - Rename `hdx_parser` to `css_parse`.
 - Extensively document `css_lexer` & `css_parse`.
 - Fold in a bunch of the macros in `hdx_ast` into `css_parse`.
- Rename `hdx_ast` to `css_ast` - make it just for CSS (the original
intent was for `hdx_ast` to have multiple language ASTs, e.g. `sass`,
but I think now we can add a `sass_ast` crate instead).

Author: keithamus

Cargo.lock

@@ -15,12 +15,10 @@ repository = "https://github.com/keithamus/hdx"
 [workspace.dependencies]
 # Local packages
 hdx = { version = "0.0.1", path = "crates/hdx" }
-hdx_atom = { version = "0.0.0", path = "crates/hdx_atom" }
 hdx_proc_macro = { version = "0.0.0", path = "crates/hdx_proc_macro" }
-hdx_parser = { version = "0.0.1", path = "crates/hdx_parser" }
-hdx_lexer = { version = "0.0.1", path = "crates/hdx_lexer" }
-hdx_syntax = { version = "0.0.0", path = "crates/hdx_syntax" }
-hdx_ast = { version = "0.0.0", path = "crates/hdx_ast" }
+css_parse = { version = "0.0.1", path = "crates/css_parse" }
+css_lexer = { version = "0.0.1", path = "crates/css_lexer" }
+css_ast = { version = "0.0.0", path = "crates/css_ast" }
 hdx_transform = { version = "0.0.0", path = "crates/hdx_transform" }
 hdx_highlight = { version = "0.0.0", path = "crates/hdx_highlight" }
 hdx_lsp = { version = "0.0.0", path = "crates/hdx_lsp" }
@@ -34,6 +32,7 @@ itertools = { version = "0.13.0" }
 ropey = { version = "1.6.1" }
 smallvec = { version = "1.13.2" }
 strum = { version = "0.26.3" }
+phf = { version = "0.11.2" }
 
 # CLI
 clap = { version = "4.5.23" }

Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "hdx_ast"
+name = "css_ast"
 version = "0.0.0"
 authors.workspace = true
 description.workspace = true
@@ -13,14 +13,14 @@ repository.workspace = true
 bench = false
 
 [dependencies]
-hdx_lexer = { workspace = true }
-hdx_atom = { workspace = true }
-hdx_parser = { workspace = true }
+css_lexer = { workspace = true }
+css_parse = { workspace = true }
 hdx_proc_macro = { workspace = true }
 
 bumpalo = { workspace = true, features = ["collections", "boxed"] }
 miette = { workspace = true, features = ["derive"] }
 smallvec = { workspace = true }
+phf = { workspace = true, features = ["macros"] }
 
 serde = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }
@@ -34,6 +34,7 @@ grep-matcher = { workspace = true }
 glob = { workspace = true }
 
 [dev-dependencies]
+css_parse = { workspace = true, features = ["testing"] }
 glob = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
@@ -47,7 +48,13 @@ pprof = { workspace = true, features = ["flamegraph", "criterion"] }
 
 [features]
 default = []
-serde = ["dep:serde", "dep:serde_json", "hdx_atom/serde", "hdx_parser/serde", "hdx_lexer/serde", "smallvec/serde"]
+serde = [
+	"dep:serde",
+	"dep:serde_json",
+	"css_parse/serde",
+	"css_lexer/serde",
+	"smallvec/serde",
+]
 fancy = ["miette/fancy-no-backtrace"]
 
 [[bench]]

crates/hdx_ast/Cargo.toml crates/css_ast/Cargo.toml

@@ -1,8 +1,8 @@
 use bumpalo::Bump;
 use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use css_ast::StyleSheet;
+use css_parse::Parser;
 use glob::glob;
-use hdx_ast::css::StyleSheet;
-use hdx_parser::{Features, Parser};
 #[cfg(target_family = "unix")]
 use pprof::criterion::{Output, PProfProfiler};
 use std::fs::read_to_string;
@@ -32,7 +32,7 @@ fn popular(c: &mut Criterion) {
 		group.bench_with_input(BenchmarkId::from_parameter(&file.name), &file.source_text, |b, source_text| {
 			b.iter_with_large_drop(|| {
 				let allocator = Bump::default();
-				let _ = Parser::new(&allocator, source_text, Features::default()).parse_entirely::<StyleSheet>();
+				let _ = Parser::new(&allocator, source_text).parse_entirely::<StyleSheet>();
 
 				allocator
 			});

crates/hdx_ast/benches/parse_popular.rs crates/css_ast/benches/parse_popular.rs

@@ -36,7 +36,7 @@ pub struct NodeMatcher<'a> {
 	stylevalue_matches: &'a mut HashSet<String>,
 }
 
-impl<'a> Sink for NodeMatcher<'a> {
+impl Sink for NodeMatcher<'_> {
 	type Error = io::Error;
 
 	fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, io::Error> {
@@ -46,13 +46,25 @@ impl<'a> Sink for NodeMatcher<'a> {
 			Err(err) => return Err(io::Error::error_message(err)),
 		};
 		self.matcher.captures_iter(mat.bytes(), &mut captures, |captures| -> bool {
+			dbg!(
+				&line,
+				&captures,
+				captures.get(2).map(|r| &line[r]),
+				captures.get(3).map(|r| &line[r]),
+				captures.get(4).map(|r| &line[r]),
+				captures.get(5).map(|r| &line[r]),
+				captures.get(6).map(|r| &line[r])
+			);
 			let value_or_visit = &line[captures.get(1).unwrap()];
-			let capture = &line[captures.get(2).unwrap()];
+			let capture = &line[captures.get(6).unwrap()];
 			if !capture.is_empty() {
 				if value_or_visit == "value" {
 					self.stylevalue_matches.insert(capture.to_string());
 				}
 				self.visit_matches.insert(capture.to_string());
+			} else {
+				dbg!(&line);
+				panic!("#[visit] or #[value] on unknown");
 			}
 			true
 		})?;
@@ -62,19 +74,39 @@ impl<'a> Sink for NodeMatcher<'a> {
 
 fn main() {
 	println!("cargo::rerun-if-changed=build.rs");
-
+	use std::time::Instant;
+	let now = Instant::now();
 	let matcher = RegexMatcherBuilder::new()
 		.multi_line(true)
 		.dot_matches_new_line(true)
-		// .build(r#"#\[value.*pub (?:struct|enum) (\w*(:?<'a>)?)"#)
+		.ignore_whitespace(true)
 		.build(
-			r#"^\s*#\[(value|visit).*?(?:pub (?:struct|enum) |(?:ranged|boolean|discrete)_feature!\()(\w*(:?<'a>)?)"#,
+			r#"
+			# match the #[value] or #[visit] attribute
+			^\s*\#\[(value|visit)
+			# munch the data between the attribute and the definition
+			.*?
+			(
+				# Is this a public definition?
+				pub\s*(?:struct|enum)\s*
+				|
+				# Or one of the parser macros that create public definitions?
+				(:?
+					keyword_set!|
+					pseudo_(?:class|element)!|
+					(?:ranged|boolean|discrete)_feature!
+				)\(
+			)
+			# munch any comments/attributes between this and our name (for macros)
+			(:?\n?\s*(:?\/\/|\#)[^\n]*)*
+			# finally grab the word (plus any lifetime definition)
+			\s*(\w*(:?<'a>)?)"#,
 		)
 		.unwrap();
 	let mut visit_matches = HashSet::new();
 	let mut stylevalue_matches = HashSet::new();
 	let mut searcher = SearcherBuilder::new().line_number(false).multi_line(true).build();
-	for entry in glob("src/css/**/*.rs").unwrap() {
+	for entry in glob("src/**/*.rs").unwrap() {
 		let str = &entry.as_ref().unwrap().display();
 		println!("cargo::rerun-if-changed={}", str);
 		let context = NodeMatcher {
@@ -121,14 +153,17 @@ fn main() {
 		}}",
 		stylevalue_matches.iter().fold(String::new(), |mut out, prop| {
 			let variant_name = prop.trim_end_matches("<'a>").trim_end_matches("StyleValue").to_string();
-			let mut atom_name = kebab(variant_name.to_owned());
-			if atom_name.starts_with("webkit") {
-				atom_name = format!("-{}", atom_name);
+			let mut variant_str = kebab(variant_name.to_owned());
+			if variant_str.starts_with("webkit") {
+				variant_str = format!("-{}", variant_str);
 			}
-			writeln!(out, "\t\t\t\t\t{}: {} = atom!(\"{}\"),", variant_name, prop, atom_name).unwrap();
+			writeln!(out, "\t\t\t\t\t{}: {} = \"{}\",", variant_name, prop, variant_str).unwrap();
 			out
 		})
 	);
 
 	let _ = write(Path::new(&env::var("OUT_DIR").unwrap()).join("css_apply_properties.rs"), source);
+
+	let elapsed = now.elapsed();
+	println!("cargo::warning=Built in {:.?}", &elapsed);
 }

crates/hdx_ast/build.rs crates/css_ast/build.rs

@@ -0,0 +1,46 @@
+mod properties;
+mod rules;
+mod selector;
+mod specificity;
+mod stylerule;
+mod stylesheet;
+mod traits;
+mod types;
+mod units;
+mod values;
+mod visit;
+
+pub use properties::*;
+pub use rules::*;
+pub use selector::*;
+pub use stylerule::*;
+pub use stylesheet::*;
+pub use types::*;
+pub use units::*;
+pub use values::*;
+pub use visit::*;
+
+use css_lexer::Span;
+use css_parse::{diagnostics, CursorSink, Parse, Parser, Result as ParserResult, ToCursors};
+
+// TODO! - delete this when we're done ;)
+#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize), serde(untagged))]
+pub enum Todo {
+	#[default]
+	Todo,
+}
+
+impl<'a> Parse<'a> for Todo {
+	fn parse(p: &mut Parser<'a>) -> ParserResult<Self> {
+		Err(diagnostics::Unimplemented(Span::new(p.offset(), p.offset())))?
+	}
+}
+
+impl ToCursors for Todo {
+	fn to_cursors(&self, _: &mut impl CursorSink) {}
+}
+
+impl<'a> Visitable<'a> for Todo {
+	fn accept<V: Visit<'a>>(&self, _: &mut V) {}
+}