unified: Switch over to using swift-syntax for parsing

ql/extractor/src/extractor.rs

@@ -29,28 +29,24 @@ pub fn run(options: Options) -> std::io::Result<()> {
                 prefix: "ql",
                 ts_language: tree_sitter_ql::LANGUAGE.into(),
                 node_types: tree_sitter_ql::NODE_TYPES,
-                desugar: None,
                 file_globs: vec!["*.ql".into(), "*.qll".into()],
             },
             simple::LanguageSpec {
                 prefix: "dbscheme",
                 ts_language: tree_sitter_ql_dbscheme::LANGUAGE.into(),
                 node_types: tree_sitter_ql_dbscheme::NODE_TYPES,
-                desugar: None,
                 file_globs: vec!["*.dbscheme".into()],
             },
             simple::LanguageSpec {
                 prefix: "json",
                 ts_language: tree_sitter_json::LANGUAGE.into(),
                 node_types: tree_sitter_json::NODE_TYPES,
-                desugar: None,
                 file_globs: vec!["*.json".into(), "*.jsonl".into(), "*.jsonc".into()],
             },
             simple::LanguageSpec {
                 prefix: "blame",
                 ts_language: tree_sitter_blame::LANGUAGE.into(),
                 node_types: tree_sitter_blame::NODE_TYPES,
-                desugar: None,
                 file_globs: vec!["*.blame".into()],
             },
         ],

ruby/extractor/src/extractor.rs

@@ -94,7 +94,9 @@ pub fn run(options: Options) -> std::io::Result<()> {
         node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?;
     let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
     let lines = lines?;
-    let source_root = std::env::current_dir().ok().and_then(|d| d.canonicalize().ok());
+    let source_root = std::env::current_dir()
+        .ok()
+        .and_then(|d| d.canonicalize().ok());
     lines
         .par_iter()
         .try_for_each(|line| {
@@ -126,7 +128,6 @@ pub fn run(options: Options) -> std::io::Result<()> {
                     &path,
                     &source,
                     &[],
-                    None,
                 );
 
                 let (ranges, line_breaks) = scan_erb(
@@ -215,7 +216,6 @@ pub fn run(options: Options) -> std::io::Result<()> {
                 &path,
                 &source,
                 &code_ranges,
-                None,
             );
             std::fs::create_dir_all(src_archive_file.parent().unwrap())?;
             if needs_conversion {

shared/tree-sitter-extractor/src/extractor/desugaring.rs

@@ -0,0 +1,104 @@
+//! Extraction for languages that rewrite their syntax tree before extraction.
+//!
+//! Unlike [`crate::extractor::simple`] (direct tree-sitter extraction), a
+//! desugaring language parses source into a [`ParsedTree`] — a `yeast::Ast`
+//! plus side-channel `extra` tokens (comments and similar) — and rewrites the
+//! AST through a [`yeast::Desugarer`] before emitting TRAP. The parser is a
+//! closure, so both tree-sitter grammars (via
+//! [`crate::extractor::tree_sitter_parser`]) and fully custom parsers plug in
+//! the same way.
+
+use crate::trap;
+use std::path::PathBuf;
+
+use crate::diagnostics;
+use crate::extractor::ParsedTree;
+use crate::extractor::driver::{self, LanguageExtractor};
+use crate::node_types::{self, NodeTypeMap};
+
+/// A parser turns source bytes into a [`ParsedTree`]. Tree-sitter grammars plug
+/// in via [`crate::extractor::tree_sitter_parser`]; custom (non-tree-sitter)
+/// parsers supply their own closure.
+pub type Parser = Box<dyn Fn(&[u8]) -> Result<ParsedTree, String> + Send + Sync>;
+
+pub struct LanguageSpec {
+    pub prefix: &'static str,
+    /// The parser: source -> `yeast::Ast` + `extra` tokens (see [`Parser`]).
+    pub parser: Parser,
+    /// Fallback TRAP schema, used only when `desugarer` does not supply its own
+    /// output schema (via [`yeast::Desugarer::output_node_types_yaml`]). May be
+    /// empty for a custom parser whose desugarer always provides the schema.
+    pub node_types: &'static str,
+    /// The desugarer applied to the parsed AST before extraction. Its
+    /// `output_node_types_yaml()` (when set) provides the TRAP schema.
+    ///
+    /// `Box<dyn yeast::Desugarer>` so the shared extractor is agnostic to the
+    /// user-defined context type the desugarer uses internally.
+    pub desugarer: Box<dyn yeast::Desugarer>,
+    pub file_globs: Vec<String>,
+}
+
+impl LanguageExtractor for LanguageSpec {
+    fn file_globs(&self) -> &[String] {
+        &self.file_globs
+    }
+
+    fn build_schema(&self) -> std::io::Result<NodeTypeMap> {
+        let effective_node_types: String = match self.desugarer.output_node_types_yaml() {
+            Some(yaml) => yeast::node_types_yaml::convert(yaml).map_err(|e| {
+                std::io::Error::other(format!(
+                    "Failed to convert YAML node-types to JSON for {}: {e}",
+                    self.prefix
+                ))
+            })?,
+            None => self.node_types.to_string(),
+        };
+        node_types::read_node_types_str(self.prefix, &effective_node_types)
+    }
+
+    fn extract_file(
+        &self,
+        schema: &NodeTypeMap,
+        diagnostics_writer: &mut diagnostics::LogWriter,
+        trap_writer: &mut trap::Writer,
+        path: &std::path::Path,
+        source: &[u8],
+    ) {
+        crate::extractor::extract_parsed(
+            self.parser.as_ref(),
+            self.prefix,
+            schema,
+            diagnostics_writer,
+            trap_writer,
+            None,
+            path,
+            source,
+            self.desugarer.as_ref(),
+        );
+    }
+}
+
+pub struct Extractor {
+    pub prefix: String,
+    pub languages: Vec<LanguageSpec>,
+    pub trap_dir: PathBuf,
+    pub source_archive_dir: PathBuf,
+    pub file_lists: Vec<PathBuf>,
+    // Typically constructed via `trap::Compression::from_env`.
+    // This allow us to report the error using our diagnostics system
+    // without exposing it to consumers.
+    pub trap_compression: Result<trap::Compression, String>,
+}
+
+impl Extractor {
+    pub fn run(&self) -> std::io::Result<()> {
+        driver::run_extractor(
+            &self.prefix,
+            &self.languages,
+            &self.trap_dir,
+            &self.source_archive_dir,
+            &self.file_lists,
+            &self.trap_compression,
+        )
+    }
+}

shared/tree-sitter-extractor/src/extractor/driver.rs

@@ -0,0 +1,210 @@
+//! Shared multi-file extraction driver.
+//!
+//! The `simple` (direct tree-sitter) and `desugaring` (parse + desugar)
+//! extractors differ only in how a language's schema is built and how a single
+//! file is extracted. Everything else — threading, matching files to languages
+//! by glob, writing TRAP, and copying into the source archive — is identical
+//! and lives here, parameterised over the [`LanguageExtractor`] trait.
+
+use globset::{GlobBuilder, GlobSetBuilder};
+use rayon::prelude::*;
+use std::fs::File;
+use std::io::BufRead;
+use std::path::{Path, PathBuf};
+
+use crate::diagnostics;
+use crate::file_paths;
+use crate::node_types::NodeTypeMap;
+use crate::trap;
+
+/// A language that [`run_extractor`] can process: it knows its file globs, its
+/// TRAP schema, and how to extract a single file. Implemented by
+/// [`super::simple::LanguageSpec`] (direct tree-sitter extraction) and
+/// [`super::desugaring::LanguageSpec`] (parse into an AST and desugar it).
+pub(crate) trait LanguageExtractor: Sync {
+    /// The file-name globs that select files for this language.
+    fn file_globs(&self) -> &[String];
+    /// Build the TRAP node-type schema used to validate emitted tuples.
+    fn build_schema(&self) -> std::io::Result<NodeTypeMap>;
+    /// Extract a single file's `source` into `trap_writer`.
+    fn extract_file(
+        &self,
+        schema: &NodeTypeMap,
+        diagnostics_writer: &mut diagnostics::LogWriter,
+        trap_writer: &mut trap::Writer,
+        path: &Path,
+        source: &[u8],
+    );
+}
+
+/// Drive extraction over `languages` for every file listed in `file_lists`.
+///
+/// Sets up the thread pool, builds a combined glob set, and for each input file
+/// dispatches to the matching language's [`LanguageExtractor::extract_file`],
+/// writing the resulting TRAP and a source-archive copy.
+pub(crate) fn run_extractor<L: LanguageExtractor>(
+    prefix: &str,
+    languages: &[L],
+    trap_dir: &Path,
+    source_archive_dir: &Path,
+    file_lists: &[PathBuf],
+    trap_compression: &Result<trap::Compression, String>,
+) -> std::io::Result<()> {
+    tracing::info!("Extraction started");
+    let diagnostics = diagnostics::DiagnosticLoggers::new(prefix);
+    let mut main_thread_logger = diagnostics.logger();
+    let num_threads = match crate::options::num_threads() {
+        Ok(num) => num,
+        Err(e) => {
+            main_thread_logger.write(
+                main_thread_logger
+                    .new_entry("configuration-error", "Configuration error")
+                    .message(
+                        "{}; defaulting to 1 thread.",
+                        &[diagnostics::MessageArg::Code(&e)],
+                    )
+                    .severity(diagnostics::Severity::Warning),
+            );
+            1
+        }
+    };
+    tracing::info!(
+        "Using {} {}",
+        num_threads,
+        if num_threads == 1 {
+            "thread"
+        } else {
+            "threads"
+        }
+    );
+    let trap_compression = match trap_compression {
+        Ok(x) => *x,
+        Err(e) => {
+            main_thread_logger.write(
+                main_thread_logger
+                    .new_entry("configuration-error", "Configuration error")
+                    .message("{}; using gzip.", &[diagnostics::MessageArg::Code(e)])
+                    .severity(diagnostics::Severity::Warning),
+            );
+            trap::Compression::Gzip
+        }
+    };
+    drop(main_thread_logger);
+
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(num_threads)
+        .build_global()
+        .unwrap();
+
+    let file_lists: Vec<File> = file_lists
+        .iter()
+        .map(|file_list| {
+            File::open(file_list)
+                .unwrap_or_else(|_| panic!("Unable to open file list at {file_list:?}"))
+        })
+        .collect();
+
+    let mut schemas = vec![];
+    for lang in languages {
+        schemas.push(lang.build_schema()?);
+    }
+
+    // Construct a single globset containing all language globs,
+    // and a mapping from glob index to language index.
+    let (globset, glob_language_mapping) = {
+        let mut builder = GlobSetBuilder::new();
+        let mut glob_lang_mapping = vec![];
+        for (i, lang) in languages.iter().enumerate() {
+            for glob_str in lang.file_globs() {
+                let glob = GlobBuilder::new(glob_str)
+                    .literal_separator(true)
+                    .build()
+                    .expect("invalid glob");
+                builder.add(glob);
+                glob_lang_mapping.push(i);
+            }
+        }
+        (
+            builder.build().expect("failed to build globset"),
+            glob_lang_mapping,
+        )
+    };
+
+    let path_transformer = file_paths::load_path_transformer()?;
+
+    let lines: std::io::Result<Vec<String>> = file_lists
+        .iter()
+        .flat_map(|file_list| std::io::BufReader::new(file_list).lines())
+        .collect();
+    let lines = lines?;
+
+    lines
+        .par_iter()
+        .try_for_each(|line| {
+            let mut diagnostics_writer = diagnostics.logger();
+            let path = PathBuf::from(line).canonicalize()?;
+            let src_archive_file = crate::file_paths::path_for(
+                source_archive_dir,
+                &path,
+                "",
+                path_transformer.as_ref(),
+            );
+            let source = std::fs::read(&path)?;
+            let mut trap_writer = trap::Writer::new();
+
+            match path.file_name() {
+                None => {
+                    tracing::error!(?path, "No file name found, skipping file.");
+                }
+                Some(filename) => {
+                    let matches = globset.matches(filename);
+                    if matches.is_empty() {
+                        tracing::error!(?path, "No matching language found, skipping file.");
+                    } else {
+                        let mut languages_processed = vec![false; languages.len()];
+
+                        for m in matches {
+                            let i = glob_language_mapping[m];
+                            if languages_processed[i] {
+                                continue;
+                            }
+                            languages_processed[i] = true;
+                            let lang = &languages[i];
+
+                            lang.extract_file(
+                                &schemas[i],
+                                &mut diagnostics_writer,
+                                &mut trap_writer,
+                                &path,
+                                &source,
+                            );
+                            std::fs::create_dir_all(src_archive_file.parent().unwrap())?;
+                            std::fs::copy(&path, &src_archive_file)?;
+                            write_trap(trap_dir, &path, &trap_writer, trap_compression)?;
+                        }
+                    }
+                }
+            }
+            Ok(()) as std::io::Result<()>
+        })
+        .expect("failed to extract files");
+
+    let path = PathBuf::from("extras");
+    let mut trap_writer = trap::Writer::new();
+    crate::extractor::populate_empty_location(&mut trap_writer);
+
+    let res = write_trap(trap_dir, &path, &trap_writer, trap_compression);
+    tracing::info!("Extraction complete");
+    res
+}
+
+fn write_trap(
+    trap_dir: &Path,
+    path: &Path,
+    trap_writer: &trap::Writer,
+    trap_compression: trap::Compression,
+) -> std::io::Result<()> {
+    let trap_file = crate::file_paths::path_for(trap_dir, path, trap_compression.extension(), None);
+    std::fs::create_dir_all(trap_file.parent().unwrap())?;
+    trap_writer.write_to_file(&trap_file, trap_compression)
+}