feat:follows linguist better now for source detection (#6)

* Follows linguist better now for source detection

* Update version

---------

Co-authored-by: ro <[email protected]>

Author: seatedro

Cargo.lock

@@ -1,9 +1,10 @@
 [package]
 name = "glimpse"
-version = "0.6.8"
+version = "0.6.9"
 edition = "2021"
 description = "A blazingly fast tool for peeking at codebases. Perfect for loading your codebase into an LLM's context."
 license = "MIT"
+build = "build.rs"
 
 [dependencies]
 anyhow = "1.0.95"
@@ -24,3 +25,7 @@ tiktoken-rs = "0.6.0"
 tokenizers = { version = "0.21.0", features = ["http"] }
 toml = "0.8.19"
 walkdir = "2.5.0"
+
+[build-dependencies]
+serde = { version = "1.0.217", features = ["derive"] }
+serde_yaml = "0.9.34"

Cargo.toml

@@ -0,0 +1,82 @@
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+#[derive(Debug, Deserialize)]
+struct Language {
+    #[serde(default)]
+    r#type: Option<String>,
+    #[serde(default)]
+    extensions: Vec<String>,
+    #[serde(default)]
+    filenames: Vec<String>,
+    #[serde(default)]
+    interpreters: Vec<String>,
+    #[serde(default)]
+    language_id: Option<i32>,
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=languages.yml");
+
+    // Read and parse languages.yml
+    let yaml_content =
+        std::fs::read_to_string("languages.yml").expect("Failed to read languages.yml");
+    let languages: HashMap<String, Language> =
+        serde_yaml::from_str(&yaml_content).expect("Failed to parse languages.yml");
+
+    // Generate the rust code
+    let mut code = String::new();
+
+    // Add the use statements
+    code.push_str("use once_cell::sync::Lazy;\n");
+    code.push_str("use std::collections::HashSet;\n\n");
+
+    // Generate source extensions set
+    code.push_str("pub static SOURCE_EXTENSIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {\n");
+    code.push_str("    let mut set = HashSet::new();\n\n");
+
+    for (_, lang) in &languages {
+        for ext in &lang.extensions {
+            let ext = ext.trim_start_matches('.');
+            code.push_str(&format!("    set.insert(\"{}\");\n", ext));
+        }
+    }
+
+    code.push_str("    set\n");
+    code.push_str("});\n\n");
+
+    // Generate filename mappings
+    code.push_str("pub static KNOWN_FILENAMES: Lazy<HashSet<&'static str>> = Lazy::new(|| {\n");
+    code.push_str("    let mut set = HashSet::new();\n\n");
+
+    for (_, lang) in &languages {
+        for filename in &lang.filenames {
+            code.push_str(&format!("    set.insert(\"{}\");\n", filename));
+        }
+    }
+
+    code.push_str("    set\n");
+    code.push_str("});\n\n");
+
+    // Generate interpreter mappings
+    code.push_str("pub static INTERPRETER_NAMES: Lazy<HashSet<&'static str>> = Lazy::new(|| {\n");
+    code.push_str("    let mut set = HashSet::new();\n\n");
+
+    for (_, lang) in &languages {
+        for interpreter in &lang.interpreters {
+            code.push_str(&format!("    set.insert(\"{}\");\n", interpreter));
+        }
+    }
+
+    code.push_str("    set\n");
+    code.push_str("});\n");
+
+    // Write the generated code to file
+    let out_dir = std::env::var_os("OUT_DIR").unwrap();
+    let dest_path = Path::new(&out_dir).join("languages.rs");
+    let mut f = File::create(dest_path).unwrap();
+    f.write_all(code.as_bytes()).unwrap();
+}

build.rs

@@ -18,7 +18,7 @@
       {
         packages.default = pkgs.rustPlatform.buildRustPackage {
           pname = "glimpse";
-          version = "0.6.8";
+          version = "0.6.9";
 
           src = ./.;