feat: progress on dbt

Author: benfdking

crates/lib/Cargo.toml

@@ -38,6 +38,7 @@ name = "depth_map"
 harness = false
 
 [features]
+default = ["python"]
 python = ["pyo3", "sqruff-lib-core/serde"]
 
 [dependencies]

crates/lib/src/core/config.rs

@@ -425,7 +425,16 @@ impl ConfigLoader {
                         unimplemented!()
                     } else if name_lowercase.ends_with("_path") || name_lowercase.ends_with("_dir")
                     {
-                        unimplemented!()
+                        // if absolute_path, just keep
+                        // if relative path, make it absolute
+                        let path = PathBuf::from(value.as_string().unwrap());
+                        if !path.is_absolute() {
+                            unimplemented!(
+                                "Relative paths are not supported yet. {}, {}",
+                                name,
+                                path.to_str().unwrap_or_default()
+                            );
+                        }
                     }
 
                     let mut key = key.clone();

crates/lib/src/templaters/dbt.rs

@@ -9,6 +9,7 @@ use pyo3::{Py, PyAny, Python};
 use sqruff_lib_core::errors::SQLFluffUserError;
 use sqruff_lib_core::templaters::base::TemplatedFile;
 use std::ffi::CString;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 pub struct DBTTemplater;
@@ -106,7 +107,7 @@ impl Templater for DBTTemplater {
             syspath.insert(0, temp_folder)?;
 
             let file_contents = CString::new(DBT_FILE).unwrap();
-            let main_module = PyModule::from_code(py, &file_contents, c"", c"")?;
+            let main_module = PyModule::from_code(py, &file_contents, &c"dbt_templater.py", &c"dbt_templater")?;
             let fun: Py<PyAny> = main_module.getattr("process_from_rust")?.into();
 
             let py_dict = config.to_python_context(py, "dbt").unwrap();
@@ -128,3 +129,64 @@ impl Templater for DBTTemplater {
         Ok(templated_file)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fmt::format;
+    use std::path::Path;
+
+    #[test]
+    fn test_dbt_simple() {
+        let templater = DBTTemplater;
+
+        let crate_location = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        let file = crate_location.join("src").join("templaters").join("dbt.rs");
+        let project_dir = file
+            .parent()
+            .unwrap()
+            .join("sqruff_templaters")
+            .join("sample_dbt")
+            .canonicalize()
+            .unwrap();
+        let profiles_dir = project_dir.join(".profiles").canonicalize().unwrap();
+        let file_path = project_dir
+            .join("models/example/my_first_dbt_model.sql")
+            .canonicalize()
+            .unwrap();
+
+        let config = format!(
+            r#"
+[sqruff]
+templater = dbt
+
+[sqruff:templater:dbt]
+project_dir = {project_dir}
+profiles_dir = {profiles_dir}
+            "#,
+            project_dir = project_dir.to_str().unwrap(),
+            profiles_dir = profiles_dir.to_str().unwrap()
+        );
+
+        let fluff_config = FluffConfig::from_source(&config, None);
+
+        let templated_file = templater
+            .process(
+                r#"
+        {{ config(materialized='table') }}
+
+        with source_data as (
+            select 1 as id
+            union all
+            select null as id
+        )
+                "#,
+                file_path.to_str().unwrap(),
+                &fluff_config,
+                &None,
+            )
+            .unwrap();
+
+        assert!(!templated_file.sliced_file.is_empty());
+    }
+}

crates/lib/src/templaters/python_shared.rs

@@ -85,11 +85,25 @@ impl From<FluffConfig> for PythonFluffConfig {
                 .unwrap_or_default(),
 
             dbt_profile: None,
-            dbt_profiles_dir: None,
+            dbt_profiles_dir: value
+                .get_section("templater")
+                .get("dbt")
+                .map(|value| value.as_map().unwrap())
+                .and_then(|value| {
+                    value
+                        .get("profiles_dir")
+                        .map(|v| v.as_string().unwrap().to_string())
+                }),
             dbt_target: None,
             dbt_target_path: None,
             dbt_context: None,
-            dbt_project_dir: None,
+            dbt_project_dir: value.get_section("templater").get("dbt").and_then(|value| {
+                value
+                    .as_map()
+                    .unwrap()
+                    .get("project_dir")
+                    .map(|v| v.as_string().unwrap().to_string())
+            }),
         }
     }
 }

crates/lib/src/templaters/sqruff_templaters/dbt_templater.py

@@ -266,14 +266,11 @@ def dbt_config(self):
 
         # Attempt to silence internal logging at this point.
         # https://github.com/sqlfluff/sqlfluff/issues/5054
-        print("dbt config")
         # self.try_silence_dbt_logs()
 
         user_config = None
         cli_vars = self._get_cli_vars()
 
-        print("set user config and cli vars")
-
         flags.set_from_args(
             DbtConfigArgs(
                 project_dir=self.project_dir,
@@ -285,7 +282,6 @@ def dbt_config(self):
             ),
             user_config,
         )
-        print("flags set")
         _dbt_config = DbtRuntimeConfig.from_args(
             DbtConfigArgs(
                 project_dir=self.project_dir,
@@ -297,10 +293,8 @@ def dbt_config(self):
                 threads=1,
             )
         )
-        print("dbt config set")
 
         register_adapter(_dbt_config, get_mp_context())
-        print("return dbt config")
         return _dbt_config
 
     @cached_property
@@ -328,8 +322,6 @@ def dbt_manifest(self):
         # dbt 0.20.* and onward
         from dbt.parser.manifest import ManifestLoader
 
-        print(self.dbt_config)
-
         return ManifestLoader.get_full_manifest(self.dbt_config)
 
     @cached_property
@@ -447,10 +439,6 @@ def sequence_files(
         if not self.profiles_dir:
             self.profiles_dir = self._get_profiles_dir()
 
-        print("project_dir", self.project_dir)
-        print("")
-        print(self)
-
         outs = []
 
         # Populate full paths for selected files
@@ -861,15 +849,12 @@ def process_from_rust(
     config_string: str,
     live_context: Dict[str, Any],
 ) -> TemplatedFile:
-    print("sys.path:", sys.path)
-    print("Current working directory:", os.getcwd())
     """Process the call from the rust side."""
     config = fluff_config_from_json(config_string)
     templater = DbtTemplater(override_context=live_context, sqlfluff_config=config)
     try:
         fnames = templater.sequence_files([fname], config=config)
         fname = fnames[0]
-        print(fname)
         (output, errors) = templater.process(
             in_str=string,
             fname=fname,
@@ -878,6 +863,7 @@ def process_from_rust(
         )
     except Exception as e:
         print("Error: ", e)
+        print("Error Stack:", e.__traceback__)
         raise e
     if errors != []:
         raise ValueError

crates/lib/test/fixtures/dbt/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

crates/lib/test/fixtures/dbt/analyses/.gitkeep

@@ -0,0 +1,36 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'test'
+version: '1.0.0'
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'test'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  test:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view

crates/lib/test/fixtures/dbt/dbt_project.yml

@@ -0,0 +1,27 @@
+
+/*
+    Welcome to your first dbt model!
+    Did you know that you can also configure models directly within SQL files?
+    This will override configurations stated in dbt_project.yml
+
+    Try changing "table" to "view" below
+*/
+
+{{ config(materialized='table') }}
+
+with source_data as (
+
+    select 1 as id
+    union all
+    select null as id
+
+)
+
+select *
+from source_data
+
+/*
+    Uncomment the line below to remove records with null `id` values
+*/
+
+-- where id is not null

crates/lib/test/fixtures/dbt/macros/.gitkeep

@@ -0,0 +1,6 @@
+
+-- Use the `ref` function to select from other models
+
+select *
+from {{ ref('my_first_dbt_model') }}
+where id = 1

crates/lib/test/fixtures/dbt/models/example/my_first_dbt_model.sql

@@ -0,0 +1,21 @@
+
+version: 2
+
+models:
+  - name: my_first_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        data_tests:
+          - unique
+          - not_null
+
+  - name: my_second_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        data_tests:
+          - unique
+          - not_null